def trylock(self):
"""Try acquiring the DB file lock without blocking
This raises BlockingIOError(11) if the other process has taken the lock.
"""
self._f.seek(0)
os.lockf(self._f.fileno(), os.F_TLOCK, 0)
def lock(self):
"""Taking an exclusive lock on the DB file
This blocks until the other process releases the lock.
"""
self._f.seek(0)
os.lockf(self._f.fileno(), os.F_LOCK, 0)
def exclusive_process(name):
# Ensure that this process, globally named `name`, does not execute
# multiple times concurrently.
import os, os.path, sys, atexit
if os.access('/var/run', os.W_OK):
# process is probably run as root
piddir = '/var/run'
else:
# process is probably a non-root user
piddir = '/tmp'
pidfile = os.path.join(piddir, '%s.pid' % name)
mypid = os.getpid()
# Attempt to get a lock on ourself so that the concurrency check
# itself is not executed in parallel.
with open(__file__, 'r+') as flock:
# Try to get a lock. This blocks until a lock is acquired. The
# lock is held until the flock file is closed at the end of the
# with block.
os.lockf(flock.fileno(), os.F_LOCK, 0)
# While we have a lock, look at the pid file. First attempt
# to write our pid to a pidfile if no file already exists there.
try:
with open(pidfile, 'x') as f:
# Successfully opened a new file. Since the file is new
# there is no concurrent process. Write our pid.
f.write(str(mypid))
# When this process exits, remove the pid file.
atexit.register(clear_my_pid, pidfile)
return
except FileExistsError:
# The pid file already exixts, but it may contain a stale
# pid of a terminated process.
with open(pidfile, 'r+') as f:
# Read the pid in the file.
existing_pid = None
try:
existing_pid = int(f.read().strip())
except ValueError:
pass # No valid integer in the file.
# Check if the pid in it is valid. If it is, there is
# already a process running. Quit.
if existing_pid:
if is_pid_valid(existing_pid):
print("Another %s process is already running (pid %d)." % (name, existing_pid), file=sys.stderr)
sys.exit(1)
# This pid is stale. Write over it with our pid.
f.seek(0)
f.write(str(mypid))
f.truncate()
# When this process exits, remove the pid file.
atexit.register(clear_my_pid, pidfile)
def __exit__(self, *exc):
try:
os.lockf(self.fd, os.F_ULOCK, 0)
except OSError:
pass
try:
self.fileobj.close()
except OSError:
pass
try:
os.remove(self.path)
except OSError:
pass
return False
def exclusive_process(name):
# Ensure that a process named `name` does not execute multiple
# times concurrently.
import os, sys, atexit
pidfile = '/var/run/mailinabox-%s.pid' % name
mypid = os.getpid()
# Attempt to get a lock on ourself so that the concurrency check
# itself is not executed in parallel.
with open(__file__, 'r+') as flock:
# Try to get a lock. This blocks until a lock is acquired. The
# lock is held until the flock file is closed at the end of the
# with block.
os.lockf(flock.fileno(), os.F_LOCK, 0)
# While we have a lock, look at the pid file. First attempt
# to write our pid to a pidfile if no file already exists there.
try:
with open(pidfile, 'x') as f:
# Successfully opened a new file. Since the file is new
# there is no concurrent process. Write our pid.
f.write(str(mypid))
atexit.register(clear_my_pid, pidfile)
return
except FileExistsError:
# The pid file already exixts, but it may contain a stale
# pid of a terminated process.
with open(pidfile, 'r+') as f:
# Read the pid in the file.
existing_pid = None
try:
existing_pid = int(f.read().strip())
except ValueError:
pass # No valid integer in the file.
# Check if the pid in it is valid.
if existing_pid:
if is_pid_valid(existing_pid):
print("Another %s is already running (pid %d)." %
(name, existing_pid),
file=sys.stderr)
sys.exit(1)
# Write our pid.
f.seek(0)
f.write(str(mypid))
f.truncate()
atexit.register(clear_my_pid, pidfile)
def main():
"""The main entry point."""
parser = argparse.ArgumentParser(
description='System call list consistency checks')
parser.add_argument('--cc',
metavar='CC',
required=True,
help='C compiler (including options) to use')
parser.add_argument('--lock',
metavar='PATH',
required=True,
help='file to lock during the updates')
parser.add_argument('arch_syscall',
metavar='ARCH-SYSCALL-H',
help='The <arch-syscall.h> file to update')
parser.add_argument('names_list',
metavar='SYSCALL-NAMES-LIST',
help='The syscall name list to update ')
args = parser.parse_args()
kernel_constants = glibcsyscalls.kernel_constants(args.cc)
with open(args.lock, 'r+') as lockfile:
os.lockf(lockfile.fileno(), os.F_LOCK, 0)
# Replace <arch-syscall.h> with data derived from kernel headers.
# No merging is necessary here. Arch-specific changes should go
# into <fixup-unistd-asm.h>.
out = io.StringIO()
out.write('/* AUTOGENERATED by update-syscall-lists.py. */\n')
for name, value in sorted(kernel_constants.items()):
out.write('#define __NR_{} {}\n'.format(name, value))
atomic_replace(args.arch_syscall, out.getvalue())
# Merge the architecture-specific system call names into the
# global names list, syscall-names.list. This file contains names
# from other architectures (and comments), so it is necessary to
# merge the existing files with the names obtained from the
# kernel.
with open(args.names_list, 'r') as list_file:
names_list = glibcsyscalls.SyscallNamesList(list_file)
merged = names_list.merge(kernel_constants.keys())
out = io.StringIO()
for line in merged:
out.write(line)
atomic_replace(args.names_list, out.getvalue())
def exclusive_process(name):
# Ensure that a process named `name` does not execute multiple
# times concurrently.
import os, sys, atexit
pidfile = "/var/run/mailinabox-%s.pid" % name
mypid = os.getpid()
# Attempt to get a lock on ourself so that the concurrency check
# itself is not executed in parallel.
with open(__file__, "r+") as flock:
# Try to get a lock. This blocks until a lock is acquired. The
# lock is held until the flock file is closed at the end of the
# with block.
os.lockf(flock.fileno(), os.F_LOCK, 0)
# While we have a lock, look at the pid file. First attempt
# to write our pid to a pidfile if no file already exists there.
try:
with open(pidfile, "x") as f:
# Successfully opened a new file. Since the file is new
# there is no concurrent process. Write our pid.
f.write(str(mypid))
atexit.register(clear_my_pid, pidfile)
return
except FileExistsError:
# The pid file already exixts, but it may contain a stale
# pid of a terminated process.
with open(pidfile, "r+") as f:
# Read the pid in the file.
existing_pid = None
try:
existing_pid = int(f.read().strip())
except ValueError:
pass # No valid integer in the file.
# Check if the pid in it is valid.
if existing_pid:
if is_pid_valid(existing_pid):
print("Another %s is already running (pid %d)." % (name, existing_pid), file=sys.stderr)
sys.exit(1)
# Write our pid.
f.seek(0)
f.write(str(mypid))
f.truncate()
atexit.register(clear_my_pid, pidfile)
def __init__(self, path, daemonize=True):
self.path = path
self.daemonize = daemonize
self.fileobj = open(self.path, "a")
self.fd = self.fileobj.fileno()
try:
os.lockf(self.fd, os.F_TLOCK, 0)
except (PermissionError, BlockingIOError):
try:
with open(self.path, "r") as lines:
for line in lines:
pid = int(line)
except (FileNotFoundError, ValueError):
raise SystemExit("%s seems to be running (unable to get pid)" % \
os.path.basename(sys.argv[0]))
except OSError as exc:
raise SystemExit(str(exc))
raise SystemExit("%s is already running as pid %s" % \
(os.path.basename(sys.argv[0]), pid))
if self.daemonize:
try:
# pylint:disable=protected-access
if os.fork() > 0:
os._exit(0)
os.setsid()
os.chdir("/")
if os.fork() > 0:
os._exit(0)
except EnvironmentError as exc:
raise SystemExit(str(exc))
fd = os.open(os.devnull, os.O_RDWR)
os.dup2(fd, 0)
os.dup2(fd, 1)
os.dup2(fd, 2)
print(os.getpid(), file=self.fileobj)
self.fileobj.flush()
def init(self):
# Create pidfile
self.fl = os.open(self.config.glob["PIDFILE"],
os.O_WRONLY + os.O_CREAT, 0o666)
try:
os.lockf(self.fl, os.F_TLOCK, 0)
os.write(self.fl, str(os.getpid()).encode())
os.fsync(self.fl)
except BlockingIOError:
os.close(self.fl)
self.fl = -1
raise JetconfInitError("Jetconf already running (pidfile exists)")
# Import backend modules
backend_package = self.config.glob["BACKEND_PACKAGE"]
try:
_be_installed = import_module(backend_package)
del _be_installed
except ImportError as e:
raise JetconfInitError(
"Cannot import backend package \"{}\", reason: {}".format(
backend_package, ErrorHelpers.epretty(e)))
try:
usr_state_data_handlers = import_module(backend_package +
".usr_state_data_handlers")
except ImportError:
usr_state_data_handlers = None
try:
usr_conf_data_handlers = import_module(backend_package +
".usr_conf_data_handlers")
except ImportError:
usr_conf_data_handlers = None
try:
usr_op_handlers = import_module(backend_package +
".usr_op_handlers")
except ImportError:
usr_op_handlers = None
try:
usr_action_handlers = import_module(backend_package +
".usr_action_handlers")
except ImportError:
usr_action_handlers = None
try:
usr_datastore = import_module(backend_package + ".usr_datastore")
except ImportError:
usr_datastore = None
try:
self.usr_init = import_module(backend_package + ".usr_init")
except ImportError:
self.usr_init = None
# Load data model
yang_mod_dir = self.config.glob["YANG_LIB_DIR"]
yang_lib_str = resource_string(
backend_package, "yang-library-data.json").decode("utf-8")
try:
datamodel = DataModel(yang_lib_str, [yang_mod_dir])
except ModuleNotFound as e:
raise JetconfInitError(
"Cannot find YANG module \"{} ({})\" in YANG library".format(
e.name, e.rev))
# Datastore init
datastore = usr_datastore.UserDatastore(
datamodel,
self.config.glob["DATA_JSON_FILE"],
with_nacm=self.config.nacm["ENABLED"])
self.datastore = datastore
try:
datastore.load()
except (FileNotFoundError, YangsonException) as e:
raise JetconfInitError(
"Cannot load JSON data file \"{}\", reason: {}".format(
self.config.glob["DATA_JSON_FILE"],
ErrorHelpers.epretty(e)))
# Validate datastore on startup
try:
datastore.get_data_root().validate(ValidationScope.all,
ContentType.config)
except (SchemaError, SemanticError) as e:
raise JetconfInitError(
"Initial validation of datastore failed, reason: {}".format(
ErrorHelpers.epretty(e)))
# Register handlers for configuration data
if usr_conf_data_handlers is not None:
usr_conf_data_handlers.register_conf_handlers(datastore)
# Register handlers for state data
if usr_state_data_handlers is not None:
usr_state_data_handlers.register_state_handlers(datastore)
# Register handlers for operations
op_internal.register_op_handlers(datastore)
if usr_op_handlers is not None:
usr_op_handlers.register_op_handlers(datastore)
# Register handlers for actions
if usr_action_handlers is not None:
usr_action_handlers.register_action_handlers(datastore)
# Init backend package
if self.usr_init is not None:
try:
self.usr_init.jc_startup()
self.backend_initiated = True
except Exception as e:
raise JetconfInitError(
"Backend initialization failed, reason: {}".format(
ErrorHelpers.epretty(e)))
# Create HTTP server
self.rest_srv = RestServer()
self.rest_srv.register_api_handlers(datastore)
def __exit__(self, etype, evalue, traceback):
os.lseek(self.f, 0, 0)
os.lockf(self.f, os.F_ULOCK, 1)
def main():
config_file = "config.yaml"
# Parse command line arguments
try:
opts, args = getopt.getopt(sys.argv[1:], "c:")
except getopt.GetoptError:
print("Invalid argument detected. Possible options are: -c (config file)")
sys.exit(1)
for opt, arg in opts:
if opt == "-c":
config_file = arg
# Load configuration
try:
load_config(config_file)
except FileNotFoundError:
print("Configuration file does not exist")
sys.exit(1)
except ParserError as e:
print("Configuration syntax error: " + str(e))
sys.exit(1)
# Set logging level
log_level = {
"error": logging.ERROR,
"warning": logging.WARNING,
"info": logging.INFO,
"debug": logging.INFO
}.get(CONFIG_GLOBAL["LOG_LEVEL"], logging.INFO)
logging.root.handlers.clear()
# Daemonize
if CONFIG_GLOBAL["LOGFILE"] not in ("-", "stdout"):
# Setup basic logging
logging.basicConfig(
format="%(asctime)s %(levelname)-8s %(message)s",
level=log_level,
filename=CONFIG_GLOBAL["LOGFILE"]
)
# Go to background
pid = os.fork()
if pid != 0:
sys.exit(0)
os.setsid()
os.umask(0)
pid = os.fork()
if pid != 0:
sys.exit(0)
# Close standard file descriptors
os.close(sys.stdin.fileno())
os.close(sys.stdout.fileno())
os.close(sys.stderr.fileno())
fd_null = os.open("/dev/null", os.O_RDWR)
os.dup(fd_null)
os.dup(fd_null)
else:
# Setup color logging
log_formatter = colorlog.ColoredFormatter(
"%(asctime)s %(log_color)s%(levelname)-8s%(reset)s %(message)s",
datefmt=None,
reset=True,
log_colors={
'DEBUG': 'cyan',
'INFO': 'green',
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'red',
},
secondary_log_colors={},
style='%'
)
log_handler = colorlog.StreamHandler()
log_handler.setFormatter(log_formatter)
log_handler.stream = sys.stdout
logger = colorlog.getLogger()
logger.addHandler(log_handler)
logger.setLevel(log_level)
# Print configuration
print_config()
# Create pidfile
fl = os.open(CONFIG_GLOBAL["PIDFILE"], os.O_WRONLY + os.O_CREAT, 0o666)
try:
os.lockf(fl, os.F_TLOCK, 0)
os.write(fl, str(os.getpid()).encode())
os.fsync(fl)
except BlockingIOError:
error("Jetconf daemon already running (pidfile exists). Exiting.")
sys.exit(1)
# Set signal handlers
def sig_exit_handler(signum, frame):
os.close(fl)
os.unlink(CONFIG_GLOBAL["PIDFILE"])
info("Exiting.")
sys.exit(0)
signal.signal(signal.SIGTERM, sig_exit_handler)
signal.signal(signal.SIGINT, sig_exit_handler)
# Load data model
yang_lib_file = os.path.join(CONFIG_GLOBAL["YANG_LIB_DIR"], "yang-library-data.json")
datamodel = DataHelpers.load_data_model(
CONFIG_GLOBAL["YANG_LIB_DIR"],
yang_lib_file
)
# Datastore init
datastore = JsonDatastore(datamodel, CONFIG_GLOBAL["DATA_JSON_FILE"], "DNS data", with_nacm=False)
try:
datastore.load()
#datastore.load_yl_data(yang_lib_file)
except (FileNotFoundError, YangsonException) as e:
error("Could not load JSON datastore " + CONFIG_GLOBAL["DATA_JSON_FILE"])
error(ErrorHelpers.epretty(e))
sig_exit_handler(0, None)
try:
datastore.get_data_root().validate(ValidationScope.all, ContentType.config)
except (SchemaError, SemanticError) as e:
error("Initial validation of datastore failed")
error(ErrorHelpers.epretty(e))
sig_exit_handler(0, None)
# Register configuration data node listeners
# TODO
# Register op handlers
# TODO
# Create and register state data node listeners
usr_state_data_handlers.create_zone_state_handlers(STATE_DATA_HANDLES, datamodel)
# datastore callbacks TODO
datastore.commit_begin_callback = tsn_connect
datastore.commit_end_callback = tsn_disconnect
# Create HTTP server
rest_srv = RestServer()
rest_srv.register_api_handlers(datastore)
rest_srv.register_static_handlers()
# Run HTTP server
rest_srv.run()
def main():
signal.signal(signal.SIGTERM, handle_sigterm)
euid = os.geteuid()
pw = pwd.getpwuid(euid)
if euid == 0: # I am root
default_config = '/etc/darbup.conf'
lock_filename = '/run/darbup.lock'
else:
if not pw.pw_dir:
sys.stderr.write('error: user {} has no home directory'.format(
pw.pw_name))
return 1
darbup_dir = os.path.join(pw.pw_dir, '.darbup')
default_config = os.path.join(darbup_dir, 'config')
lock_filename = os.path.join(darbup_dir, 'lock')
try:
os.mkdir(darbup_dir)
except FileExistsError:
pass
parser = argparse.ArgumentParser(description='regular backup using dar')
parser.add_argument('--full', action='store_true', help='do full backup')
parser.add_argument('--incr', action='store_true',
help='do incremental backup')
parser.add_argument('-c', '--config', metavar='FILENAME',
help='configuration file to use (default: {})' \
.format(default_config), default=default_config)
parser.add_argument('-l', '--loglevel', metavar='LEVEL',
help='logging level (default: INFO)',
choices=('DEBUG','INFO','WARNING','ERROR','CRITICAL'),
default='INFO')
args = parser.parse_args()
if args.full and args.incr:
sys.stderr.write(parser.format_usage())
sys.stderr.write('error: only one of --full, --incr may be given\n')
return 2
have_lock = False
try:
with open(lock_filename, 'a+') as lock_file:
lock_file.seek(0)
try:
os.lockf(lock_file.fileno(), os.F_TLOCK, 0)
except BlockingIOError:
other_pid = lock_file.read().rstrip()
sys.stderr.write('Another instance of darbup is already '
'running as PID {} for user {}\n'
.format(other_pid, pw.pw_name))
return 1
have_lock = True
lock_file.truncate(0)
lock_file.write('{}\n'.format(os.getpid()))
lock_file.flush()
darbup(args, default_config)
finally:
if have_lock:
os.remove(lock_filename)
def exclusive_process(name):
# Ensure that this process, globally named `name`, does not execute
# multiple times concurrently.
import os, os.path, sys, atexit
if os.access('/var/run', os.W_OK):
# process is probably run as root
piddir = '/var/run'
else:
# process is probably a non-root user
piddir = '/tmp'
pidfile = os.path.join(piddir, '%s.pid' % name)
mypid = os.getpid()
# Attempt to get a lock on ourself so that the concurrency check
# itself is not executed in parallel.
with open(__file__, 'r+') as flock:
# Try to get a lock. This blocks until a lock is acquired. The
# lock is held until the flock file is closed at the end of the
# with block.
os.lockf(flock.fileno(), os.F_LOCK, 0)
# While we have a lock, look at the pid file. First attempt
# to write our pid to a pidfile if no file already exists there.
try:
with open(pidfile, 'x') as f:
# Successfully opened a new file. Since the file is new
# there is no concurrent process. Write our pid.
f.write(str(mypid))
# When this process exits, remove the pid file.
atexit.register(clear_my_pid, pidfile)
return
except FileExistsError:
# The pid file already exixts, but it may contain a stale
# pid of a terminated process.
with open(pidfile, 'r+') as f:
# Read the pid in the file.
existing_pid = None
try:
existing_pid = int(f.read().strip())
except ValueError:
pass # No valid integer in the file.
# Check if the pid in it is valid. If it is, there is
# already a process running. Quit.
if existing_pid:
if is_pid_valid(existing_pid):
print(
"Another %s process is already running (pid %d)." %
(name, existing_pid),
file=sys.stderr)
sys.exit(1)
# This pid is stale. Write over it with our pid.
f.seek(0)
f.write(str(mypid))
f.truncate()
# When this process exits, remove the pid file.
atexit.register(clear_my_pid, pidfile)
def _acquire(self):
self.lockfile = get_lock_file(self.name)
my_pid = str(os.getpid())
# Write our process ID to the lockfile, if the lockfile doesn't
# yet exist.
try:
with open(self.lockfile, 'x') as f:
# Successfully opened a new file. Since the file is new
# there is no concurrent process. Write our pid. Lock
# first to prevent a race condition with the next block.
os.lockf(f.fileno(), os.F_LOCK, 0)
f.write(my_pid)
except FileExistsError:
# The lockfile already exists, or at least it did at the
# moment we tried to open it above. That probably indicates
# another process is running and is holding the lock.
#
# But it may contain a stale pid of a terminated process.
#
# And the file may have been deleted in a race condition.
# In that case, an OSError will probably be raised, which
# we'll re-wrap as a CannotAcquireLock.
#
# Open the lockfile in update ("r+") mode so we can check
# the PID inside it and, if the PID is stale, write ours
# to the file.
try:
with open(self.lockfile, 'r+') as f:
# Get an exclusive lock. This blocks until a lock is acquired. The
# lock is held until the flock file is closed at the end of the
# with block.
os.lockf(f.fileno(), os.F_LOCK, 0)
# Read the pid in the file.
try:
existing_pid = int(f.read().strip())
except ValueError:
# The contents of the lockfile are not valid.
pass
else:
# Check if the pid in it is valid, and if so, the
# lock is held by another process and this Lock
# cannot be created.
if is_pid_valid(existing_pid):
msg = "Another '%s' process is already running (pid %d)." % (self.name, existing_pid)
if not self.die:
raise CannotAcquireLock(msg)
else:
print(msg, file=sys.stderr)
sys.exit(1)
# The file didn't have a valid pid, so overwrite the file
# with our pid.
f.seek(0)
f.write(my_pid)
f.truncate()
except OSError as e:
# There was a problem opening the existing lock file.
raise CannotAcquireLock("There was an error opening %s after an open in 'x' mode failed, which might indicate the lock was held just moments ago: %s." % (self.lockfile, str(e)))
# Log success. Can't do this before the open since we expect
# it to fail sometimes.
logging.info("Acquired lock at " + self.lockfile + "...")
def run_simulation_test(basedir, options):
config = ""
binary_ext = ""
if sys.platform == "win32":
config = options.config
binary_ext = ".exe"
fdbserver = os.path.realpath(
os.path.join(basedir, "bin", config, "fdbserver" + binary_ext))
pargs = [fdbserver, "-r", options.testtype]
seed = 0
if options.seed is not None:
pargs.append("-s")
seed = int(options.seed, 0)
if options.test_number:
idx = int(options.test_number)
seed = ((seed + idx) % (2**32 - 2)) + 1
pargs.append("{}".format(seed))
if options.testtype == "test":
pargs.append("-C")
pargs.append(os.path.join(args.builddir, "fdb.cluster"))
td = TestDirectory(basedir)
if options.buggify:
pargs.append("-b")
pargs.append("on")
if options.crash:
pargs.append("--crash")
# Use old style argument with underscores because old binaries don't support hyphens
pargs.append("--trace_format")
pargs.append(options.log_format)
test_dir = td.get_current_test_dir()
if options.seed is not None:
seed = int(options.seed, 0)
if options.test_number:
idx = int(options.test_number)
seed = ((seed + idx) % (2**32 - 2)) + 1
wd = os.path.join(test_dir,
"test_{}".format(options.name.replace("/", "_")))
os.mkdir(wd)
return_codes = {} # {command: return_code}
first = True
restart_test_policy = None
if len(options.testfile) > 1:
restart_test_policy = RestartTestPolicy(options.testfile[0],
options.old_binary, fdbserver)
for testfile in options.testfile:
tmp = list(pargs)
valgrind_args = []
if restart_test_policy is not None:
if first:
tmp[0] = restart_test_policy.first_binary()
else:
tmp[0] = restart_test_policy.second_binary()
# old_binary is not under test, so don't run under valgrind
if options.use_valgrind and tmp[0] == fdbserver:
valgrind_args = ["valgrind", "--error-exitcode=99", "--"]
if not first:
tmp.append("-R")
if seed is not None:
seed = (seed + 1) % (2**32 - 2)
first = False
if seed is not None:
tmp.append("-s")
tmp.append("{}".format(seed))
tmp.append("-f")
tmp.append(testfile)
tmp = valgrind_args + tmp
command = " ".join(tmp)
_logger.info("COMMAND: {}".format(command))
proc = subprocess.Popen(tmp,
stdout=sys.stdout,
stderr=sys.stderr,
cwd=wd)
proc.wait()
return_codes[command] = proc.returncode
outfile = os.path.join(test_dir,
"traces.{}".format(options.log_format))
res = True
if options.aggregate_traces == "NONE":
res = process_traces(
basedir,
options.name,
wd,
None,
"NONE",
options.symbolicate,
options.log_format,
return_codes,
options.seed,
)
else:
with open(outfile, "a") as f:
os.lockf(f.fileno(), os.F_LOCK, 0)
pos = f.tell()
res = process_traces(
basedir,
options.name,
wd,
f,
options.aggregate_traces,
options.symbolicate,
options.log_format,
return_codes,
options.seed,
)
f.seek(pos)
os.lockf(f.fileno(), os.F_ULOCK, 0)
if proc.returncode != 0 or not res:
break
if options.keep_logs == "NONE" or options.keep_logs == "FAILED" and res:
print("Deleting old logs in {}".format(wd))
traces = get_traces(wd, options.log_format)
for trace in traces:
os.remove(trace)
if options.keep_simdirs == "NONE" or options.keep_simdirs == "FAILED" and res:
print("Delete {}".format(os.path.join(wd, "simfdb")))
# Don't fail if the directory doesn't exist.
try:
shutil.rmtree(os.path.join(wd, "simfdb"))
except FileNotFoundError:
pass
if len(os.listdir(wd)) == 0:
print("Delete {} - empty".format(wd))
os.rmdir(wd)
return res and proc.returncode == 0
def unlock(self):
"""Release the file lock taken in IPAddrDb.lock()"""
self._f.seek(0)
os.lockf(self._f.fileno(), os.F_ULOCK, 0)
def __enter__(self):
self.fp = open(self.filename, "ab")
os.lockf(self.fp.fileno(), os.F_LOCK, 0)
def run_simulation_test(basedir, options):
fdbserver = os.path.join(basedir, 'bin', 'fdbserver')
pargs = [fdbserver, '-r', options.testtype]
if options.testtype == 'test':
pargs.append('-C')
pargs.append(os.path.join(args.builddir, 'fdb.cluster'))
else:
pargs.append('-S')
pargs.append('off')
td = TestDirectory(basedir)
if options.buggify:
pargs.append('-b')
pargs.append('on')
pargs.append('--trace_format')
pargs.append(options.log_format)
test_dir = td.get_current_test_dir()
if options.seed is not None:
pargs.append('-s')
seed = int(options.seed, 0)
if options.test_number:
idx = int(options.test_number)
seed = ((seed + idx) % (2**32 - 2)) + 1
pargs.append("{}".format(seed))
wd = os.path.join(test_dir,
'test_{}'.format(options.name.replace('/', '_')))
os.mkdir(wd)
return_codes = {} # {command: return_code}
first = True
for testfile in options.testfile:
tmp = list(pargs)
if first and options.old_binary is not None and len(
options.testfile) > 1:
_logger.info("Run old binary at {}".format(options.old_binary))
tmp[0] = options.old_binary
if not first:
tmp.append('-R')
first = False
tmp.append('-f')
tmp.append(testfile)
command = ' '.join(tmp)
_logger.info("COMMAND: {}".format(command))
proc = subprocess.Popen(tmp,
stdout=sys.stdout,
stderr=sys.stderr,
cwd=wd)
proc.wait()
return_codes[command] = proc.returncode
if proc.returncode != 0:
break
outfile = os.path.join(test_dir, 'traces.{}'.format(options.log_format))
res = True
if options.aggregate_traces == 'NONE':
res = process_traces(basedir, options.name, wd, None, 'NONE',
options.log_format, return_codes, options.seed)
else:
with open(outfile, 'a') as f:
os.lockf(f.fileno(), os.F_LOCK, 0)
pos = f.tell()
res = process_traces(basedir, options.name, wd, f,
options.aggregate_traces, options.log_format,
return_codes, options.seed)
f.seek(pos)
os.lockf(f.fileno(), os.F_ULOCK, 0)
if options.keep_logs == 'NONE' or options.keep_logs == 'FAILED' and res:
print("Deleting old logs in {}".format(wd))
traces = get_traces(wd, options.log_format)
for trace in traces:
os.remove(trace)
if options.keep_simdirs == 'NONE' or options.keep_simdirs == 'FAILED' and res:
print("Delete {}".format(os.path.join(wd, 'simfdb')))
shutil.rmtree(os.path.join(wd, 'simfdb'))
if len(os.listdir(wd)) == 0:
print("Delete {} - empty".format(wd))
os.rmdir(wd)
return res
def __exit__(self, *args):
if self.fp is not None:
os.lockf(self.fp.fileno(), os.F_ULOCK, 0)
self.fp.close()
def run_simulation_test(basedir,
testtype,
testname,
testfiles,
log_format,
restart=False,
buggify=False,
seed=None):
pargs = [os.path.join(basedir, 'bin', 'fdbserver'),
'-r', testtype]
if testtype == 'test':
pargs.append('-C')
pargs.append(os.path.join(args.builddir, 'fdb.cluster'))
td = TestDirectory(basedir)
if restart:
pargs.append('-R')
if buggify:
pargs.append('-b')
pargs.append('on')
# FIXME: include these lines as soon as json support is added
#pargs.append('--trace_format')
#pargs.append(log_format)
test_dir = td.get_current_test_dir()
if seed is not None:
pargs.append('-s')
pargs.append(str(args.seed))
wd = os.path.join(test_dir,
'test_{}'.format(testname.replace('/', '_')))
os.mkdir(wd)
return_codes = {} # {command: return_code}
first = True
for testfile in testfiles:
tmp = list(pargs)
if not first:
tmp.append('-R')
first = False
tmp.append('-f')
tmp.append(testfile)
command = ' '.join(tmp)
_logger.info("COMMAND: {}".format(command))
proc = subprocess.Popen(tmp,
stdout=sys.stdout,
stderr=sys.stderr,
cwd=wd)
proc.wait()
return_codes[command] = proc.returncode
if proc.returncode != 0:
break
outfile = os.path.join(test_dir, 'traces.{}'.format(log_format))
res = True
with open(outfile, 'a') as f:
os.lockf(f.fileno(), os.F_LOCK, 0)
pos = f.tell()
res = process_traces(basedir, testname,
os.path.join(os.getcwd(), wd), f, log_format,
return_codes)
f.seek(pos)
os.lockf(f.fileno(), os.F_ULOCK, 0)
if res:
shutil.rmtree(wd)
return res
def run_simulation_test(basedir, options):
fdbserver = os.path.join(basedir, 'bin', 'fdbserver')
pargs = [fdbserver, '-r', options.testtype]
seed = 0
if options.seed is not None:
pargs.append('-s')
seed = int(options.seed, 0)
if options.test_number:
idx = int(options.test_number)
seed = ((seed + idx) % (2**32 - 2)) + 1
pargs.append("{}".format(seed))
if options.testtype == 'test':
pargs.append('-C')
pargs.append(os.path.join(args.builddir, 'fdb.cluster'))
td = TestDirectory(basedir)
if options.buggify:
pargs.append('-b')
pargs.append('on')
if options.crash:
pargs.append('--crash')
pargs.append('--trace_format')
pargs.append(options.log_format)
test_dir = td.get_current_test_dir()
if options.seed is not None:
seed = int(options.seed, 0)
if options.test_number:
idx = int(options.test_number)
seed = ((seed + idx) % (2**32 - 2)) + 1
wd = os.path.join(test_dir,
'test_{}'.format(options.name.replace('/', '_')))
os.mkdir(wd)
return_codes = {} # {command: return_code}
first = True
restart_test_policy = None
if len(options.testfile) > 1:
restart_test_policy = RestartTestPolicy(options.testfile[0],
options.old_binary, fdbserver)
for testfile in options.testfile:
tmp = list(pargs)
valgrind_args = []
if restart_test_policy is not None:
if first:
tmp[0] = restart_test_policy.first_binary()
else:
tmp[0] = restart_test_policy.second_binary()
# old_binary is not under test, so don't run under valgrind
if options.use_valgrind and tmp[0] == fdbserver:
valgrind_args = ['valgrind', '--error-exitcode=99', '--']
if not first:
tmp.append('-R')
if seed is not None:
seed = ((seed + 1) % (2**32 - 2))
first = False
if seed is not None:
tmp.append('-s')
tmp.append("{}".format(seed))
tmp.append('-f')
tmp.append(testfile)
tmp = valgrind_args + tmp
command = ' '.join(tmp)
_logger.info("COMMAND: {}".format(command))
proc = subprocess.Popen(tmp,
stdout=sys.stdout,
stderr=sys.stderr,
cwd=wd)
proc.wait()
return_codes[command] = proc.returncode
outfile = os.path.join(test_dir,
'traces.{}'.format(options.log_format))
res = True
if options.aggregate_traces == 'NONE':
res = process_traces(basedir, options.name, wd, None, 'NONE',
options.symbolicate, options.log_format,
return_codes, options.seed)
else:
with open(outfile, 'a') as f:
os.lockf(f.fileno(), os.F_LOCK, 0)
pos = f.tell()
res = process_traces(basedir, options.name, wd, f,
options.aggregate_traces,
options.symbolicate, options.log_format,
return_codes, options.seed)
f.seek(pos)
os.lockf(f.fileno(), os.F_ULOCK, 0)
if proc.returncode != 0 or res == False:
break
if options.keep_logs == 'NONE' or options.keep_logs == 'FAILED' and res:
print("Deleting old logs in {}".format(wd))
traces = get_traces(wd, options.log_format)
for trace in traces:
os.remove(trace)
if options.keep_simdirs == 'NONE' or options.keep_simdirs == 'FAILED' and res:
print("Delete {}".format(os.path.join(wd, 'simfdb')))
# Don't fail if the directory doesn't exist.
try:
shutil.rmtree(os.path.join(wd, 'simfdb'))
except FileNotFoundError:
pass
if len(os.listdir(wd)) == 0:
print("Delete {} - empty".format(wd))
os.rmdir(wd)
return res
def main():
config_file = "config.yaml"
# Check for Python version
if sys.version_info < (3, 5):
print("Jetconf requires Python version 3.5 or higher")
sys.exit(1)
# Get Jetconf version
jetconf_version = get_distribution("jetconf").version
# Parse command line arguments
try:
opts, args = getopt.getopt(sys.argv[1:], "c:vh")
except getopt.GetoptError:
print("Error: invalid argument detected.")
print_help()
sys.exit(1)
for opt, arg in opts:
if opt == "-c":
config_file = arg
elif opt == "-v":
print("Jetconf version {}".format(jetconf_version))
sys.exit(0)
elif opt == "-h":
print_help()
sys.exit(0)
# Load configuration
try:
load_config(config_file)
except FileNotFoundError:
print("Configuration file does not exist")
sys.exit(1)
except ParserError as e:
print("Configuration syntax error: " + str(e))
sys.exit(1)
# Validate configuration
try:
validate_config()
except ValueError as e:
print("Error: " + str(e))
sys.exit(1)
# Set logging level
log_level = {
"error": logging.ERROR,
"warning": logging.WARNING,
"info": logging.INFO,
"debug": logging.INFO
}.get(CONFIG_GLOBAL["LOG_LEVEL"], logging.INFO)
logging.root.handlers.clear()
# Daemonize
if CONFIG_GLOBAL["LOGFILE"] not in ("-", "stdout"):
# Setup basic logging
logging.basicConfig(format="%(asctime)s %(levelname)-8s %(message)s",
level=log_level,
filename=CONFIG_GLOBAL["LOGFILE"])
# Go to background
pid = os.fork()
if pid != 0:
sys.exit(0)
os.setsid()
os.umask(0)
pid = os.fork()
if pid != 0:
sys.exit(0)
# Close standard file descriptors
os.close(sys.stdin.fileno())
os.close(sys.stdout.fileno())
os.close(sys.stderr.fileno())
fd_null = os.open("/dev/null", os.O_RDWR)
os.dup(fd_null)
os.dup(fd_null)
else:
# Setup color logging
log_formatter = colorlog.ColoredFormatter(
"%(asctime)s %(log_color)s%(levelname)-8s%(reset)s %(message)s",
datefmt=None,
reset=True,
log_colors={
'DEBUG': 'cyan',
'INFO': 'green',
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'red',
},
secondary_log_colors={},
style='%')
log_handler = colorlog.StreamHandler()
log_handler.setFormatter(log_formatter)
log_handler.stream = sys.stdout
logger = colorlog.getLogger()
logger.addHandler(log_handler)
logger.setLevel(log_level)
# Print version
info("Jetconf version {}".format(jetconf_version))
# Print configuration
print_config()
# Create pidfile
fl = os.open(CONFIG_GLOBAL["PIDFILE"], os.O_WRONLY + os.O_CREAT, 0o666)
try:
os.lockf(fl, os.F_TLOCK, 0)
os.write(fl, str(os.getpid()).encode())
os.fsync(fl)
except BlockingIOError:
error("Jetconf daemon already running (pidfile exists). Exiting.")
sys.exit(1)
# Set signal handlers
def sig_exit_handler(signum, frame):
os.close(fl)
os.unlink(CONFIG_GLOBAL["PIDFILE"])
info("Exiting.")
sys.exit(0)
signal.signal(signal.SIGTERM, sig_exit_handler)
signal.signal(signal.SIGINT, sig_exit_handler)
# Import backend modules
backend_package = CONFIG_GLOBAL["BACKEND_PACKAGE"]
try:
usr_state_data_handlers = import_module(backend_package +
".usr_state_data_handlers")
usr_conf_data_handlers = import_module(backend_package +
".usr_conf_data_handlers")
usr_op_handlers = import_module(backend_package + ".usr_op_handlers")
usr_datastore = import_module(backend_package + ".usr_datastore")
except ImportError as e:
error(ErrorHelpers.epretty(e))
error("Cannot import backend package \"{}\". Exiting.".format(
backend_package))
sys.exit(1)
# Load data model
yang_mod_dir = CONFIG_GLOBAL["YANG_LIB_DIR"]
yang_lib_str = resource_string(backend_package,
"yang-library-data.json").decode("utf-8")
datamodel = DataModel(yang_lib_str, [yang_mod_dir])
# Datastore init
datastore = usr_datastore.UserDatastore(datamodel,
CONFIG_GLOBAL["DATA_JSON_FILE"],
with_nacm=CONFIG_NACM["ENABLED"])
try:
datastore.load()
except (FileNotFoundError, YangsonException) as e:
error("Could not load JSON datastore " +
CONFIG_GLOBAL["DATA_JSON_FILE"])
error(ErrorHelpers.epretty(e))
sig_exit_handler(0, None)
# Validate datastore on startup
try:
datastore.get_data_root().validate(ValidationScope.all,
ContentType.config)
except (SchemaError, SemanticError) as e:
error("Initial validation of datastore failed")
error(ErrorHelpers.epretty(e))
sig_exit_handler(0, None)
# Register handlers for configuration data
usr_conf_data_handlers.register_conf_handlers(datastore)
# Register handlers for state data
usr_state_data_handlers.register_state_handlers(datastore)
# Register handlers for operations
op_internal.register_op_handlers(datastore)
usr_op_handlers.register_op_handlers(datastore)
# Create HTTP server
rest_srv = RestServer()
rest_srv.register_api_handlers(datastore)
rest_srv.register_static_handlers()
# Run HTTP server
rest_srv.run()
def __enter__(self):
os.lseek(self.f, 0, 0)
return os.lockf(self.f, os.F_LOCK if self.block else os.F_TLOCK,
1) == 0
