def init(self):
self.in_dir = os.environ['MFDATA_DATA_IN_DIR']
conf_file = os.path.join(os.environ['MODULE_RUNTIME_HOME'], "tmp",
"config_auto", "switch.ini")
if not os.path.exists(conf_file):
self.error_and_die("no switch configuration file")
self.condition_tuples = []
parser = ExtendedConfigParser(config=CONFIG,
strict=False,
inheritance='im',
interpolation=None)
parser.read(conf_file)
sections = parser.sections()
for section in sections:
directory = parser.get(section, "directory").strip()
plugin_name = parser.get(section, "plugin_name").strip()
condition = parser.get(section, "condition").strip()
magic_file = parser.get(section, "magic_file").strip()
use_hardlink = parser.getboolean(section, "use_hardlink")
if magic_file.lower() in ('null', 'none'):
magic_file = None
self.condition_tuples.append(
(plugin_name, condition, directory, magic_file, use_hardlink))
self.no_match_policy = self.args.no_match_policy
if self.no_match_policy not in ('keep', 'delete', 'move'):
self.error_and_die("unknown no-match policy: %s",
self.no_match_policy)
if self.no_match_policy == "move":
nmpmdd = self.args.no_match_policy_move_dest_dir
if nmpmdd is None:
self.error_and_die('you have to set a '
'no-match-policy-move-dest-dir'
' in case of move no-match policy')
mkdir_p_or_die(nmpmdd)
def _init(self):
super(AcquisitionStep, self)._init()
self.failure_policy = self.args.failure_policy
self.failure_policy_move_dest_dir = None
self.failure_policy_move_absolute_dir = False
if self.failure_policy not in ("keep", "delete", "move"):
self.error_and_die("unknown failure policy: %s",
self.failure_policy)
if self.failure_policy == "move":
fpmdd = self.args.failure_policy_move_dest_dir
if fpmdd is None or fpmdd == "" or fpmdd == "FIXME":
self.error_and_die("you have to set a "
"failure-policy-move-dest-dir"
" in case of move failure policy")
if "/" not in fpmdd:
self.error_and_die(
"failure-policy-move-dest-dir must be something like "
"plugin_name/step_name or an absolute path")
if fpmdd.startswith('/'):
# absolute dir
mkdir_p_or_die(fpmdd)
self.add_virtual_trace(fpmdd)
self.failure_policy_move_dest_dir = fpmdd
self.failure_policy_move_absolute_dir = True
else:
# plugin_name/step_name
plugin_name = fpmdd.split('/')[0]
step_name = fpmdd.split('/')[1]
self.add_virtual_trace(plugin_name, step_name)
self.failure_policy_move_dest_dir = \
get_plugin_step_directory_path(plugin_name, step_name)
mkdir_p_or_die(self.failure_policy_move_dest_dir)
signal.signal(signal.SIGTERM, self.__sigterm_handler)
def init(self):
if self.args.dest_dir is None:
raise Exception('you have to set a dest-dir')
mkdir_p_or_die(self.args.dest_dir)
self.keep_tags = self.args.keep_tags
self.keep_tags_suffix = self.args.keep_tags_suffix
self.drop_tags = self.args.drop_tags
def init(self):
if self.args.dest_dir is None:
module_runtime_home = os.environ.get('MODULE_RUNTIME_HOME', '/tmp')
self.archive_dir = os.path.join(module_runtime_home, 'var',
'archive')
else:
self.archive_dir = self.args.dest_dir
self.strftime_template = self.args.strftime_template
self.keep_tags = self.args.keep_tags
self.keep_tags_suffix = self.args.keep_tags_suffix
mkdir_p_or_die(self.archive_dir)
def _init(self):
super(AcquisitionStep, self)._init()
self.failure_policy = self.args.failure_policy
if self.failure_policy not in ("keep", "delete", "move"):
self.error_and_die("unknown failure policy: %s",
self.failure_policy)
if self.failure_policy == "move":
fpmdd = self.args.failure_policy_move_dest_dir
if fpmdd is None:
self.error_and_die("you have to set a "
"failure-policy-move-dest-dir"
" in case of move failure policy")
mkdir_p_or_die(fpmdd)
self.failure_policy_move_keep_tags = (
self.args.failure_policy_move_keep_tags)
self.failure_policy_move_keep_tags_suffix = (
self.args.failure_policy_move_keep_tags_suffix)
signal.signal(signal.SIGTERM, self.__sigterm_handler)
def init_plugins_base(plugins_base_dir=None):
"""Initialize the plugins base.
Args:
plugins_base_dir (string): alternate plugins base directory
(useful for unit tests).
Raises:
MFUtilPluginCantInit: if we can't init the plugin base.
"""
plugins_base_dir = _get_plugins_base_dir(plugins_base_dir)
shutil.rmtree(plugins_base_dir, ignore_errors=True)
mkdir_p_or_die(plugins_base_dir)
mkdir_p_or_die(os.path.join(plugins_base_dir, "base"))
cmd = _get_rpm_cmd("--initdb", plugins_base_dir=plugins_base_dir)
BashWrapperOrRaise(cmd, MFUtilPluginCantInit,
"can't init %s" % plugins_base_dir)
def init_plugins_base(plugins_base_dir=None):
"""Initialize the plugins base.
Args:
plugins_base_dir (string): alternate plugins base directory
(useful for unit tests).
Raises:
MFUtilPluginCantInit: if we can't init the plugin base.
"""
plugins_base_dir = _get_plugins_base_dir(plugins_base_dir)
shutil.rmtree(plugins_base_dir, ignore_errors=True)
mkdir_p_or_die(plugins_base_dir)
mkdir_p_or_die(os.path.join(plugins_base_dir, "base"))
cmd = _get_rpm_cmd("--initdb", plugins_base_dir=plugins_base_dir)
BashWrapperOrRaise(cmd, MFUtilPluginCantInit,
"can't init %s" % plugins_base_dir)
def __init__(self,
plugins_base_dir=None,
configuration_class=None,
app_class=None,
extra_daemon_class=ExtraDaemon):
self.configuration_class = get_configuration_class(
configuration_class, Configuration)
"""Configuration class."""
self.app_class = get_app_class(app_class, App)
"""App class."""
self.extra_daemon_class = get_extra_daemon_class(
extra_daemon_class, ExtraDaemon)
"""ExtraDaemon class."""
self.plugins_base_dir = plugins_base_dir \
if plugins_base_dir is not None else get_default_plugins_base_dir()
"""Plugin base directory (string)."""
if not os.path.isdir(self.plugins_base_dir):
mkdir_p_or_die(self.plugins_base_dir)
self.__loaded = False
def _init(self):
parser = self.__get_argument_parser()
if self.unit_tests and self.unit_tests_args:
self.args = parser.parse_args(self.unit_tests_args)
else:
self.args = parser.parse_args()
self.failure_policy = self.args.failure_policy
if self.failure_policy not in ('keep', 'delete', 'move'):
self.error_and_die("unknown failure policy: %s",
self.failure_policy)
if self.failure_policy == 'move':
fpmdd = self.args.failure_policy_move_dest_dir
if fpmdd is None:
self.error_and_die('you have to set a '
'failure-policy-move-dest-dir'
' in case of move failure policy')
mkdir_p_or_die(fpmdd)
signal.signal(signal.SIGTERM, self.__sigterm_handler)
self.init()
def _init(self):
AcquisitionStep._init(self)
self.__xafs = {}
if self.args.dest_dir is None:
raise Exception('you have to set a dest-dir argument')
if self.args.dest_dir == "FIXME":
raise Exception('you have to set a dest-dir argument')
if '/' not in self.args.dest_dir:
raise Exception("invalid dest_dir: %s" % self.args.dest_dir)
if self.args.dest_dir.startswith('/'):
raise Exception("invalid dest_dir: %s, must be something like: "
"plugin_name/step_name" % self.args.dest_dir)
plugin_name = self.args.dest_dir.split('/')[0]
step_name = self.args.dest_dir.split('/')[1]
self.add_virtual_trace(plugin_name, step_name)
self.add_virtual_trace(">delete", "giveup")
self.dest_dir = get_plugin_step_directory_path(plugin_name, step_name)
mkdir_p_or_die(self.dest_dir)
self.retry_total = self.args.retry_total
self.retry_min_wait = self.args.retry_min_wait
self.retry_max_wait = self.args.retry_max_wait
self.retry_backoff = self.args.retry_backoff
def _init(self):
AcquisitionStep._init(self)
if self.args.dest_dir == "FIXME":
raise Exception("dest_dir is not set")
if '/' not in self.args.dest_dir:
raise Exception("invalid dest_dir: %s" % self.args.dest_dir)
if self.args.dest_dir.startswith('/'):
self.dest_dir = self.args.dest_dir
else:
plugin_name = self.args.dest_dir.split('/')[0]
step_name = self.args.dest_dir.split('/')[1]
self.dest_dir = get_plugin_step_directory_path(plugin_name,
step_name)
mkdir_p_or_die(self.dest_dir)
self.add_virtual_trace(self.dest_dir)
try:
if self.args.drop_tags == "AUTO":
self.drop_tags = self.args.dest_dir.startswith('/')
elif self.args.drop_tags == "1":
self.drop_tags = True
elif self.args.drop_tags == "0":
self.drop_tags = False
else:
raise Exception("invalid value for drop_tags configuration "
"key: %s" % self.args.drop_tags)
except AttributeError:
self.drop_tags = True
try:
if self.args.force_chmod == "null":
self.force_chmod = ""
else:
self.force_chmod = self.args.force_chmod
except AttributeError:
self.force_chmod = ""
if self.args.dest_basename == "":
raise Exception("dest_basename can't be empty")
self.dest_basename = self.args.dest_basename
def build_plugin(plugin_path, plugins_base_dir=None):
plugin_path = os.path.abspath(plugin_path)
plugins_base_dir = _get_plugins_base_dir(plugins_base_dir)
base = os.path.join(plugins_base_dir, "base")
pwd = os.getcwd()
parser = ExtendedConfigParser(config=os.environ.get('MFCONFIG', 'GENERIC'),
strict=False,
inheritance="im")
with open(os.path.join(plugin_path, "config.ini"), "r") as f:
config_content = f.read()
if six.PY2:
parser.read_string(config_content.decode('utf-8'))
else:
parser.read_string(config_content)
name = parser['general']['name']
version = parser['general']['version']
summary = parser['general']['summary']
license = parser['general']['license']
try:
packager = parser['general']['packager']
except Exception:
packager = parser['general']['maintainer']
vendor = parser['general']['vendor']
url = parser['general']['url']
tmpdir = os.path.join(RUNTIME_HOME, "tmp",
"plugin_%s" % get_unique_hexa_identifier())
mkdir_p_or_die(os.path.join(tmpdir, "BUILD"))
mkdir_p_or_die(os.path.join(tmpdir, "RPMS"))
mkdir_p_or_die(os.path.join(tmpdir, "SRPMS"))
_make_plugin_spec(os.path.join(tmpdir, "specfile.spec"), name, version,
summary, license, packager, vendor, url)
cmd = "source %s/lib/bash_utils.sh ; " % MFEXT_HOME
cmd = cmd + "layer_load rpm@mfext ; "
cmd = cmd + 'rpmbuild --define "_topdir %s" --define "pwd %s" ' \
'--define "prefix %s" --dbpath %s ' \
'-bb %s/specfile.spec' % (tmpdir, plugin_path, tmpdir,
base, tmpdir)
x = BashWrapperOrRaise(cmd, MFUtilPluginCantBuild,
"can't build plugin %s" % plugin_path)
tmp = glob.glob(os.path.join(tmpdir, "RPMS", "x86_64", "*.rpm"))
if len(tmp) == 0:
raise MFUtilPluginCantBuild("can't find generated plugin" %
plugin_path,
bash_wrapper=x)
plugin_path = tmp[0]
new_basename = \
os.path.basename(plugin_path).replace("x86_64.rpm",
"metwork.%s.plugin" %
MODULE_LOWERCASE)
new_plugin_path = os.path.join(pwd, new_basename)
shutil.move(plugin_path, new_plugin_path)
shutil.rmtree(tmpdir, True)
os.chdir(pwd)
return new_plugin_path
def build_plugin(plugin_path, plugins_base_dir=None):
plugin_path = os.path.abspath(plugin_path)
plugins_base_dir = _get_plugins_base_dir(plugins_base_dir)
base = os.path.join(plugins_base_dir, "base")
pwd = os.getcwd()
parser = ExtendedConfigParser(config=os.environ.get('MFCONFIG', 'GENERIC'),
strict=False, inheritance="im")
with open(os.path.join(plugin_path, "config.ini"), "r") as f:
config_content = f.read()
if six.PY2:
parser.read_string(config_content.decode('utf-8'))
else:
parser.read_string(config_content)
name = parser['general']['name']
version = parser['general']['version']
summary = parser['general']['summary']
license = parser['general']['license']
try:
packager = parser['general']['packager']
except Exception:
packager = parser['general']['maintainer']
vendor = parser['general']['vendor']
url = parser['general']['url']
tmpdir = os.path.join(RUNTIME_HOME, "tmp",
"plugin_%s" % get_unique_hexa_identifier())
mkdir_p_or_die(os.path.join(tmpdir, "BUILD"))
mkdir_p_or_die(os.path.join(tmpdir, "RPMS"))
mkdir_p_or_die(os.path.join(tmpdir, "SRPMS"))
_make_plugin_spec(os.path.join(tmpdir, "specfile.spec"), name, version,
summary, license, packager, vendor, url)
cmd = "source %s/lib/bash_utils.sh ; " % MFEXT_HOME
cmd = cmd + "layer_load rpm@mfext ; "
cmd = cmd + 'rpmbuild --define "_topdir %s" --define "pwd %s" ' \
'--define "prefix %s" --dbpath %s ' \
'-bb %s/specfile.spec' % (tmpdir, plugin_path, tmpdir,
base, tmpdir)
x = BashWrapperOrRaise(cmd, MFUtilPluginCantBuild,
"can't build plugin %s" % plugin_path)
tmp = glob.glob(os.path.join(tmpdir, "RPMS", "x86_64", "*.rpm"))
if len(tmp) == 0:
raise MFUtilPluginCantBuild("can't find generated plugin" %
plugin_path, bash_wrapper=x)
plugin_path = tmp[0]
new_basename = \
os.path.basename(plugin_path).replace("x86_64.rpm",
"metwork.%s.plugin" %
MODULE_LOWERCASE)
new_plugin_path = os.path.join(pwd, new_basename)
shutil.move(plugin_path, new_plugin_path)
shutil.rmtree(tmpdir, True)
os.chdir(pwd)
return new_plugin_path
def build_plugin(plugin_path, plugins_base_dir=None):
"""Build a plugin.
Args:
plugin_path (string): the plugin path to build
plugins_base_dir (string): (optional) the plugin base directory path.
If not set, the default plugins base directory path is used.
Raises:
MFUtilPluginCantBuild: if a error occurs during build
"""
plugin_path = os.path.abspath(plugin_path)
plugins_base_dir = _get_plugins_base_dir(plugins_base_dir)
base = os.path.join(plugins_base_dir, "base")
pwd = os.getcwd()
parser = OpinionatedConfigParser()
with open(os.path.join(plugin_path, "config.ini"), "r") as f:
config_content = f.read()
if six.PY2:
parser.read_string(config_content.decode('utf-8'))
else:
parser.read_string(config_content)
with open(os.path.join(plugin_path, ".layerapi2_label"), "r") as f:
name = f.read().replace('plugin_', '', 1).split('@')[0]
version = parser['general']['version']
summary = parser['general']['summary']
license = parser['general']['license']
try:
packager = parser['general']['packager']
except Exception:
packager = parser['general']['maintainer']
vendor = parser['general']['vendor']
url = parser['general']['url']
tmpdir = os.path.join(RUNTIME_HOME, "tmp",
"plugin_%s" % get_unique_hexa_identifier())
mkdir_p_or_die(os.path.join(tmpdir, "BUILD"))
mkdir_p_or_die(os.path.join(tmpdir, "RPMS"))
mkdir_p_or_die(os.path.join(tmpdir, "SRPMS"))
_make_plugin_spec(os.path.join(tmpdir, "specfile.spec"), name, version,
summary, license, packager, vendor, url)
cmd = "source %s/lib/bash_utils.sh ; " % MFEXT_HOME
cmd = cmd + "layer_load rpm@mfext ; "
cmd = cmd + 'rpmbuild --define "_topdir %s" --define "pwd %s" ' \
'--define "prefix %s" --dbpath %s ' \
'-bb %s/specfile.spec' % (tmpdir, plugin_path, tmpdir,
base, tmpdir)
x = BashWrapperOrRaise(cmd, MFUtilPluginCantBuild,
"can't build plugin %s" % plugin_path)
tmp = glob.glob(os.path.join(tmpdir, "RPMS", "x86_64", "*.rpm"))
if len(tmp) == 0:
raise MFUtilPluginCantBuild("can't find generated plugin" %
plugin_path, bash_wrapper=x)
plugin_path = tmp[0]
new_basename = \
os.path.basename(plugin_path).replace("x86_64.rpm",
"metwork.%s.plugin" %
MFMODULE_LOWERCASE)
new_plugin_path = os.path.join(pwd, new_basename)
shutil.move(plugin_path, new_plugin_path)
shutil.rmtree(tmpdir, True)
os.chdir(pwd)
return new_plugin_path
def init(self):
self.__xafs = {}
if self.args.reinject_dir is None:
self.error_and_die('you have to set a reinject-dir')
mkdir_p_or_die(self.args.reinject_dir)
def _get_or_make_trash_dir(plugin_name, step_name):
trash_dir = os.path.join(TRASH_DIR, plugin_name + "." + step_name)
mkdir_p_or_die(trash_dir)
return trash_dir
def _get_or_make_tmp_dir(plugin_name, step_name):
tmp_dir = os.path.join(TMP_DIR, plugin_name + "." + step_name)
mkdir_p_or_die(tmp_dir)
return tmp_dir
except NotInstalledPlugin:
echo_bold("ERROR: not installed plugin: " % args.PLUGIN_NAME)
sys.exit(1)
if plugin.is_dev_linked:
sys.exit(0)
conf_file = "%s/config.ini" % plugin.home
try:
with open(conf_file, "r") as f:
lines = f.readlines()
except Exception as e:
echo_bold("ERROR: can't read %s, exception: %s" % (conf_file, e))
sys.exit(1)
mkdir_p_or_die("%s/config/plugins" % MFMODULE_RUNTIME_HOME)
target = "%s/config/plugins/%s.ini" % (MFMODULE_RUNTIME_HOME,
args.PLUGIN_NAME)
if os.path.exists(target):
if has_an_interesting_line(target):
# we prefer not to override
new_target = "%s.new" % target
LOGGER.warning("%s already exists, we prefer not to override, "
"new file created as: %s => please merge them "
"manually" % (target, new_target))
target = new_target
with open(target, "w") as f:
f.write("# THIS FILE OVERRIDES %s CONFIGURATION FILE (plugin: %s)\n" %
(conf_file, args.PLUGIN_NAME))
def init(self):
if self.args.dest_dir is None:
self.error_and_die('you have to set a dest-dir')
mkdir_p_or_die(self.args.dest_dir)
def init(self):
super().init()
if self.args.netcdf_dest_dir is None:
raise Exception('you have to set a netcdf-dest-dir')
mkdir_p_or_die(self.args.netcdf_dest_dir)
def get_plugin_lock_path():
lock_dir = os.path.join(MFMODULE_RUNTIME_HOME, 'tmp')
lock_path = os.path.join(lock_dir, "plugin_management_lock")
if not os.path.isdir(lock_dir):
mkdir_p_or_die(lock_dir)
return lock_path
def init(self):
if self.args.dest_dir is None:
raise Exception('you have to set a dest-dir')
mkdir_p_or_die(self.args.dest_dir)
self.failure_policy = "delete"
def init(self):
if self.args.dest_dir is None:
self.error_and_die('you have to set a dest-dir')
mkdir_p_or_die(self.args.dest_dir)
self.cmodule = self._get_compression_module()
