def do_revision(config, cmd):
'''Generate new revision files, one per branch.'''
addn_kwargs = {
'message': CONF.command.message,
'autogenerate': CONF.command.autogenerate,
'sql': CONF.command.sql,
}
if _use_separate_migration_branches(CONF):
for branch in MIGRATION_BRANCHES:
version_path = _get_version_branch_path(CONF, branch)
addn_kwargs['version_path'] = version_path
if not os.path.exists(version_path):
# Bootstrap initial directory structure
utils.ensure_dir(version_path)
# Each new release stream of migrations is detached from
# previous migration chains
addn_kwargs['head'] = 'base'
# Mark the very first revision in the new branch with its label
addn_kwargs['branch_label'] = _get_branch_label(branch)
# TODO(ihrachyshka): ideally, we would also add depends_on here
# to refer to the head of the previous release stream. But
# alembic API does not support it yet.
else:
addn_kwargs['head'] = _get_branch_head(branch)
do_alembic_command(config, cmd, **addn_kwargs)
else:
do_alembic_command(config, cmd, **addn_kwargs)
update_heads_file(config)
def _get_state_file_path(self, loadbalancer_id, kind, ensure_state_dir=True):
"""Returns the file name for a given kind of config file."""
confs_dir = os.path.abspath(os.path.normpath(self.state_path))
conf_dir = os.path.join(confs_dir, loadbalancer_id)
if ensure_state_dir:
n_utils.ensure_dir(conf_dir)
return os.path.join(conf_dir, kind)
def initialize_map(self):
# Create a default table if one is not already found
self.etc.create()
utils.ensure_dir(os.path.dirname(self._rt_tables_filename))
if not os.path.exists(self._rt_tables_filename):
self._write_map(self.DEFAULT_TABLES)
self._keep = set()
def _check_bootstrap_new_branch(branch, version_path, addn_kwargs):
addn_kwargs['version_path'] = version_path
addn_kwargs['head'] = _get_branch_head(branch)
if not os.path.exists(version_path):
# Bootstrap initial directory structure
utils.ensure_dir(version_path)
addn_kwargs['branch_label'] = branch
def __init__(self, conf, network, process_monitor, version=None,
plugin=None):
super(DhcpLocalProcess, self).__init__(conf, network, process_monitor,
version, plugin)
self.confs_dir = self.get_confs_dir(conf)
self.network_conf_dir = os.path.join(self.confs_dir, network.id)
commonutils.ensure_dir(self.network_conf_dir)
def __init__(self,
conf,
uuid,
namespace=None,
service=None,
pids_path=None,
default_cmd_callback=None,
cmd_addl_env=None,
pid_file=None,
run_as_root=False):
self.conf = conf
self.uuid = uuid
self.namespace = namespace
self.default_cmd_callback = default_cmd_callback
self.cmd_addl_env = cmd_addl_env
self.pids_path = pids_path or self.conf.external_pids
self.pid_file = pid_file
self.run_as_root = run_as_root
if service:
self.service_pid_fname = 'pid.' + service
self.service = service
else:
self.service_pid_fname = 'pid'
self.service = 'default-service'
common_utils.ensure_dir(os.path.dirname(self.get_pid_file_name()))
def _check_bootstrap_new_branch(branch, version_path, addn_kwargs):
addn_kwargs['version_path'] = version_path
addn_kwargs['head'] = _get_branch_head(branch)
if not os.path.exists(version_path):
# Bootstrap initial directory structure
utils.ensure_dir(version_path)
addn_kwargs['branch_label'] = branch
def __init__(
self,
conf,
uuid,
namespace=None,
service=None,
pids_path=None,
default_cmd_callback=None,
cmd_addl_env=None,
pid_file=None,
run_as_root=False,
):
self.conf = conf
self.uuid = uuid
self.namespace = namespace
self.default_cmd_callback = default_cmd_callback
self.cmd_addl_env = cmd_addl_env
self.pids_path = pids_path or self.conf.external_pids
self.pid_file = pid_file
self.run_as_root = run_as_root
if service:
self.service_pid_fname = "pid." + service
self.service = service
else:
self.service_pid_fname = "pid"
self.service = "default-service"
common_utils.ensure_dir(os.path.dirname(self.get_pid_file_name()))
def __init__(self, conf, network, process_monitor, version=None,
plugin=None):
super(DhcpLocalProcess, self).__init__(conf, network, process_monitor,
version, plugin)
self.confs_dir = self.get_confs_dir(conf)
self.network_conf_dir = os.path.join(self.confs_dir, network.id)
commonutils.ensure_dir(self.network_conf_dir)
def do_revision(config, cmd):
'''Generate new revision files, one per branch.'''
addn_kwargs = {
'message': CONF.command.message,
'autogenerate': CONF.command.autogenerate,
'sql': CONF.command.sql,
}
if _use_separate_migration_branches(config):
for branch in MIGRATION_BRANCHES:
version_path = _get_version_branch_path(config, branch)
addn_kwargs['version_path'] = version_path
if not os.path.exists(version_path):
# Bootstrap initial directory structure
utils.ensure_dir(version_path)
# Each new release stream of migrations is detached from
# previous migration chains
addn_kwargs['head'] = 'base'
# Mark the very first revision in the new branch with its label
addn_kwargs['branch_label'] = _get_branch_label(branch)
# TODO(ihrachyshka): ideally, we would also add depends_on here
# to refer to the head of the previous release stream. But
# alembic API does not support it yet.
else:
addn_kwargs['head'] = _get_branch_head(branch)
do_alembic_command(config, cmd, **addn_kwargs)
else:
do_alembic_command(config, cmd, **addn_kwargs)
update_heads_file(config)
def initialize_map(self):
# Create a default table if one is not already found
self.etc.create()
utils.ensure_dir(os.path.dirname(self._rt_tables_filename))
if not os.path.exists(self._rt_tables_filename):
self._write_map(self.DEFAULT_TABLES)
self._keep = set()
def _get_conf_base(cfg_root, uuid, ensure_conf_dir):
#TODO(mangelajo): separate responsibilities here, ensure_conf_dir
# should be a separate function
conf_dir = os.path.abspath(os.path.normpath(cfg_root))
conf_base = os.path.join(conf_dir, uuid)
if ensure_conf_dir:
utils.ensure_dir(conf_dir)
return conf_base
def setup_test_logging(config_opts, log_dir, log_file_path_template):
# Have each test log into its own log file
config_opts.set_override('debug', True)
utils.ensure_dir(log_dir)
log_file = sanitize_log_path(os.path.join(log_dir, log_file_path_template))
config_opts.set_override('log_file', log_file)
config_opts.set_override('use_stderr', False)
config.setup_logging()
def _get_state_file_path(self, loadbalancer_id, kind,
ensure_state_dir=True):
"""Returns the file name for a given kind of config file."""
confs_dir = os.path.abspath(os.path.normpath(self.state_path))
conf_dir = os.path.join(confs_dir, loadbalancer_id)
if ensure_state_dir:
n_utils.ensure_dir(conf_dir)
return os.path.join(conf_dir, kind)
def enable(self):
"""Enables DHCP for this network by spawning a local process."""
if self.active:
self.restart()
elif self._enable_dhcp():
commonutils.ensure_dir(self.network_conf_dir)
interface_name = self.device_manager.setup(self.network)
self.interface_name = interface_name
self.spawn_process()
def setup_test_logging(config_opts, log_dir, log_file_path_template):
# Have each test log into its own log file
config_opts.set_override('debug', True)
utils.ensure_dir(log_dir)
log_file = sanitize_log_path(
os.path.join(log_dir, log_file_path_template))
config_opts.set_override('log_file', log_file)
config_opts.set_override('use_stderr', False)
config.setup_logging()
def enable(self):
"""Enables DHCP for this network by spawning a local process."""
if self.active:
self.restart()
elif self._enable_dhcp():
commonutils.ensure_dir(self.network_conf_dir)
interface_name = self.device_manager.setup(self.network)
self.interface_name = interface_name
self.spawn_process()
def start(self):
fmt = self.process_name + "--%Y-%m-%d--%H%M%S.log"
log_dir = os.path.join(DEFAULT_LOG_DIR, self.test_name)
common_utils.ensure_dir(log_dir)
cmd = [spawn.find_executable(self.exec_name), "--log-dir", log_dir, "--log-file", timeutils.strtime(fmt=fmt)]
for filename in self.config_filenames:
cmd += ["--config-file", filename]
self.process = async_process.AsyncProcess(cmd)
self.process.start(block=True)
def _get_allocations(self):
utils.ensure_dir(TMP_DIR)
try:
with open(self._state_file_path, 'r') as allocations_file:
contents = allocations_file.read()
except IOError:
contents = None
# If the file was empty, we want to return an empty set, not {''}
return set(contents.split(',')) if contents else set()
def _get_allocations(self):
utils.ensure_dir(TMP_DIR)
try:
with open(self._state_file_path, 'r') as allocations_file:
contents = allocations_file.read()
except IOError:
contents = None
# If the file was empty, we want to return an empty set, not {''}
return set(contents.split(',')) if contents else set()
def ensure_directory_exists_without_file(path):
dirname = os.path.dirname(path)
if os.path.isdir(dirname):
try:
os.unlink(path)
except OSError:
with excutils.save_and_reraise_exception() as ctxt:
if not os.path.exists(path):
ctxt.reraise = False
else:
utils.ensure_dir(dirname)
def start(self):
log_dir = os.path.join(DEFAULT_LOG_DIR, self.test_name)
common_utils.ensure_dir(log_dir)
timestamp = datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S-%f")
log_file = "%s--%s.log" % (self.process_name, timestamp)
cmd = [spawn.find_executable(self.exec_name),
'--log-dir', log_dir,
'--log-file', log_file]
for filename in self.config_filenames:
cmd += ['--config-file', filename]
self.process = async_process.AsyncProcess(cmd)
self.process.start(block=True)
def start(self):
log_dir = os.path.join(DEFAULT_LOG_DIR, self.test_name)
common_utils.ensure_dir(log_dir)
timestamp = datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S-%f")
log_file = "%s--%s.log" % (self.process_name, timestamp)
cmd = [spawn.find_executable(self.exec_name),
'--log-dir', log_dir,
'--log-file', log_file]
for filename in self.config_filenames:
cmd += ['--config-file', filename]
self.process = async_process.AsyncProcess(cmd)
self.process.start(block=True)
def start(self):
fmt = self.process_name + "--%Y-%m-%d--%H%M%S.log"
log_dir = os.path.join(DEFAULT_LOG_DIR, self.test_name)
common_utils.ensure_dir(log_dir)
cmd = [
spawn.find_executable(self.exec_name), '--log-dir', log_dir,
'--log-file',
datetime.utcnow().strftime(fmt)
]
for filename in self.config_filenames:
cmd += ['--config-file', filename]
self.process = async_process.AsyncProcess(cmd)
self.process.start(block=True)
def __init__(self, host=None):
super(DhcpAgent, self).__init__(host=host)
self.needs_resync_reasons = collections.defaultdict(list)
self.conf = cfg.CONF
self.cache = NetworkCache()
self.dhcp_driver_cls = importutils.import_class(self.conf.dhcp_driver)
ctx = context.get_admin_context_without_session()
self.plugin_rpc = DhcpPluginApi(topics.PLUGIN, ctx,
self.conf.use_namespaces)
# create dhcp dir to store dhcp info
dhcp_dir = os.path.dirname("/%s/dhcp/" % self.conf.state_path)
utils.ensure_dir(dhcp_dir)
self.dhcp_version = self.dhcp_driver_cls.check_version()
self._populate_networks_cache()
self._process_monitor = external_process.ProcessMonitor(
config=self.conf, resource_type='dhcp')
def __init__(self, host=None, conf=None):
super(DhcpAgent, self).__init__(host=host)
self.needs_resync_reasons = collections.defaultdict(list)
self.conf = conf or cfg.CONF
self.cache = NetworkCache()
self.dhcp_driver_cls = importutils.import_class(self.conf.dhcp_driver)
ctx = context.get_admin_context_without_session()
self.plugin_rpc = DhcpPluginApi(topics.PLUGIN, ctx, self.conf.host)
# create dhcp dir to store dhcp info
dhcp_dir = os.path.dirname("/%s/dhcp/" % self.conf.state_path)
utils.ensure_dir(dhcp_dir)
self.dhcp_version = self.dhcp_driver_cls.check_version()
self._populate_networks_cache()
self._process_monitor = external_process.ProcessMonitor(
config=self.conf,
resource_type='dhcp')
def __init__(self, host=None, conf=None):
super(DhcpAgent, self).__init__(host=host)
self.needs_resync_reasons = collections.defaultdict(list)
self.dhcp_ready_ports = set()
self.conf = conf or cfg.CONF
self.cache = NetworkCache()
self.dhcp_driver_cls = importutils.import_class(self.conf.dhcp_driver)
self.plugin_rpc = DhcpPluginApi(topics.PLUGIN, self.conf.host)
# create dhcp dir to store dhcp info
dhcp_dir = os.path.dirname("/%s/dhcp/" % self.conf.state_path)
utils.ensure_dir(dhcp_dir)
self.dhcp_version = self.dhcp_driver_cls.check_version()
self._populate_networks_cache()
# keep track of mappings between networks and routers for
# metadata processing
self._metadata_routers = {} # {network_id: router_id}
self._process_monitor = external_process.ProcessMonitor(
config=self.conf, resource_type='dhcp')
def start(self):
test_name = base.sanitize_log_path(self.test_name)
log_dir = os.path.join(fullstack_base.DEFAULT_LOG_DIR, test_name)
common_utils.ensure_dir(log_dir)
timestamp = datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S-%f")
log_file = "%s--%s.log" % (self.process_name, timestamp)
cmd = [spawn.find_executable(self.exec_name),
'--log-dir', log_dir,
'--log-file', log_file]
for filename in self.config_filenames:
cmd += ['--config-file', filename]
run_as_root = bool(self.namespace)
self.process = async_process.AsyncProcess(
cmd, run_as_root=run_as_root, namespace=self.namespace
)
self.process.start(block=True)
def setUp(self):
super(BaseSudoTestCase, self).setUp()
if not base.bool_from_env('OS_SUDO_TESTING'):
self.skipTest('Testing with sudo is not enabled')
# Have each test log into its own log file
cfg.CONF.set_override('debug', True)
common_utils.ensure_dir(DEFAULT_LOG_DIR)
log_file = base.sanitize_log_path(
os.path.join(DEFAULT_LOG_DIR, "%s.log" % self.id()))
cfg.CONF.set_override('log_file', log_file)
config.setup_logging()
config.register_root_helper(cfg.CONF)
self.config(group='AGENT',
root_helper=os.environ.get('OS_ROOTWRAP_CMD', SUDO_CMD))
self.config(group='AGENT',
root_helper_daemon=os.environ.get(
'OS_ROOTWRAP_DAEMON_CMD'))
def __init__(self, host=None, conf=None):
super(DhcpAgent, self).__init__(host=host)
self.needs_resync_reasons = collections.defaultdict(list)
self.dhcp_ready_ports = set()
self.conf = conf or cfg.CONF
self.cache = NetworkCache()
self.dhcp_driver_cls = importutils.import_class(self.conf.dhcp_driver)
self.plugin_rpc = DhcpPluginApi(topics.PLUGIN, self.conf.host)
# create dhcp dir to store dhcp info
dhcp_dir = os.path.dirname("/%s/dhcp/" % self.conf.state_path)
utils.ensure_dir(dhcp_dir)
self.dhcp_version = self.dhcp_driver_cls.check_version()
self._populate_networks_cache()
# keep track of mappings between networks and routers for
# metadata processing
self._metadata_routers = {} # {network_id: router_id}
self._process_monitor = external_process.ProcessMonitor(
config=self.conf,
resource_type='dhcp')
def setUp(self):
super(BaseSudoTestCase, self).setUp()
if not base.bool_from_env('OS_SUDO_TESTING'):
self.skipTest('Testing with sudo is not enabled')
# Have each test log into its own log file
cfg.CONF.set_override('debug', True)
common_utils.ensure_dir(DEFAULT_LOG_DIR)
log_file = base.sanitize_log_path(
os.path.join(DEFAULT_LOG_DIR, "%s.log" % self.id()))
cfg.CONF.set_override('log_file', log_file)
config.setup_logging()
config.register_root_helper(cfg.CONF)
self.config(group='AGENT',
root_helper=os.environ.get('OS_ROOTWRAP_CMD', SUDO_CMD))
self.config(
group='AGENT',
root_helper_daemon=os.environ.get('OS_ROOTWRAP_DAEMON_CMD'))
def do_revision(config, cmd):
"""Generate new revision files, one per branch."""
addn_kwargs = {"message": CONF.command.message, "autogenerate": CONF.command.autogenerate, "sql": CONF.command.sql}
if _use_separate_migration_branches(config):
for branch in MIGRATION_BRANCHES:
version_path = _get_version_branch_path(config, branch)
addn_kwargs["version_path"] = version_path
addn_kwargs["head"] = _get_branch_head(branch)
if not os.path.exists(version_path):
# Bootstrap initial directory structure
utils.ensure_dir(version_path)
# Mark the very first revision in the new branch with its label
addn_kwargs["branch_label"] = branch
do_alembic_command(config, cmd, **addn_kwargs)
else:
do_alembic_command(config, cmd, **addn_kwargs)
update_heads_file(config)
def do_revision(config, cmd):
'''Generate new revision files, one per branch.'''
addn_kwargs = {
'message': CONF.command.message,
'autogenerate': CONF.command.autogenerate,
'sql': CONF.command.sql,
}
if _use_separate_migration_branches(config):
for branch in MIGRATION_BRANCHES:
version_path = _get_version_branch_path(config, branch)
addn_kwargs['version_path'] = version_path
addn_kwargs['head'] = _get_branch_head(branch)
if not os.path.exists(version_path):
# Bootstrap initial directory structure
utils.ensure_dir(version_path)
# Mark the very first revision in the new branch with its label
addn_kwargs['branch_label'] = branch
do_alembic_command(config, cmd, **addn_kwargs)
else:
do_alembic_command(config, cmd, **addn_kwargs)
update_heads_file(config)
def ensure_config_dir(self, vpnservice):
"""Create config directory if it does not exist."""
n_utils.ensure_dir(self.config_dir)
for subdir in self.CONFIG_DIRS:
dir_path = os.path.join(self.config_dir, subdir)
n_utils.ensure_dir(dir_path)
def ensure_config_dir(self, vpnservice):
"""Create config directory if it does not exist."""
n_utils.ensure_dir(self.config_dir)
for subdir in self.CONFIG_DIRS:
dir_path = os.path.join(self.config_dir, subdir)
n_utils.ensure_dir(dir_path)
def test_ensure_dir_calls_makedirs(self, makedirs):
utils.ensure_dir("/etc/create/directory")
makedirs.assert_called_once_with("/etc/create/directory", 0o755)
def test_ensure_dir_no_fail_if_exists(self, makedirs):
error = OSError()
error.errno = errno.EEXIST
makedirs.side_effect = error
utils.ensure_dir("/etc/create/concurrently")
def test_ensure_dir_calls_makedirs(self, makedirs):
utils.ensure_dir("/etc/create/directory")
makedirs.assert_called_once_with("/etc/create/directory", 0o755)
def test_ensure_dir_no_fail_if_exists(self, makedirs):
error = OSError()
error.errno = errno.EEXIST
makedirs.side_effect = error
utils.ensure_dir("/etc/create/concurrently")
def get_full_config_file_path(self, filename, ensure_conf_dir=True):
conf_dir = self.get_conf_dir()
if ensure_conf_dir:
common_utils.ensure_dir(conf_dir)
return os.path.join(conf_dir, filename)
def get_GreenletProfiler_trace_socket_path(cls):
path = os.path.join('/tmp', str(os.getpid()), 'sock')
utils.ensure_dir(os.path.dirname(path))
return path
def ensure_dir(*args, **kwargs):
return utils.ensure_dir(*args, **kwargs)
def _init_ha_conf_path(self):
ha_full_path = os.path.dirname("/%s/" % self.conf.ha_confs_path)
common_utils.ensure_dir(ha_full_path)
def _init_ha_conf_path(self):
ha_full_path = os.path.dirname("/%s/" % self.conf.ha_confs_path)
common_utils.ensure_dir(ha_full_path)
def get_full_config_file_path(self, filename, ensure_conf_dir=True):
conf_dir = self.get_conf_dir()
if ensure_conf_dir:
common_utils.ensure_dir(conf_dir)
return os.path.join(conf_dir, filename)
def ensure_dir(*args, **kwargs):
return utils.ensure_dir(*args, **kwargs)
