def _setup_cloudbiolinux(options):
def fabricrc_loader(env):
_setup_cloudbiolinux_fabric_properties(env, options)
flavor = get_main_options_string(options, "flavor", DEFAULT_CLOUDBIOLINUX_FLAVOR)
_setup_logging(env)
_configure_fabric_environment(env, flavor, fabricrc_loader=fabricrc_loader)
def install_biolinux(target=None, flavor=None):
"""Main entry point for installing BioLinux on a remote server.
`flavor` allows customization of CloudBioLinux behavior. It can either
be a flavor name that maps to a corresponding directory in contrib/flavor
or the path to a custom directory. This can contain:
- alternative package lists (main.yaml, packages.yaml, custom.yaml)
- custom python code (nameflavor.py) that hooks into the build machinery
`target` allows running only particular parts of the build process. Valid choices are:
- packages Install distro packages
- custom Install custom packages
- chef_recipes Provision chef recipes
- libraries Install programming language libraries
- post_install Setup CloudMan, FreeNX and other system services
- cleanup Remove downloaded files and prepare images for AMI builds
"""
_setup_logging(env)
time_start = _print_time_stats("Config", "start")
_check_fabric_version()
if env.ssh_config_path and os.path.isfile(os.path.expanduser(env.ssh_config_path)):
env.use_ssh_config = True
_configure_fabric_environment(env, flavor,
ignore_distcheck=(target is not None
and target in ["libraries", "custom"]))
env.logger.debug("Target is '%s'" % target)
env.logger.debug("Flavor is '%s'" % flavor)
_perform_install(target, flavor)
_print_time_stats("Config", "end", time_start)
if hasattr(env, "keep_isolated") and env.keep_isolated:
_print_shell_exports(env)
def install_biolinux(target=None, flavor=None):
"""Main entry point for installing BioLinux on a remote server.
`flavor` allows customization of CloudBioLinux behavior. It can either
be a flavor name that maps to a corresponding directory in contrib/flavor
or the path to a custom directory. This can contain:
- alternative package lists (main.yaml, packages.yaml, custom.yaml)
- custom python code (nameflavor.py) that hooks into the build machinery
`target` allows running only particular parts of the build process. Valid choices are:
- packages Install distro packages
- custom Install custom packages
- libraries Install programming language libraries
- post_install Setup CloudMan, FreeNX and other system services
- cleanup Remove downloaded files and prepare images for AMI builds
"""
_setup_logging(env)
time_start = _print_time_stats("Config", "start")
_check_fabric_version()
_configure_fabric_environment(env, flavor)
env.logger.debug("Target is '%s'" % target)
_perform_install(target, flavor)
_print_time_stats("Config", "end", time_start)
def deploy(options):
_setup_logging(env)
actions = _expand_actions(options.get("actions"))
if options["vm_provider"] == "novm":
vm_launcher = LocalVmLauncher(options)
else:
if not build_vm_launcher:
raise ImportError("Require vmlauncher: https://github.com/jmchilton/vm-launcher")
vm_launcher = build_vm_launcher(options)
if _do_perform_action("list", actions):
for node in vm_launcher.list():
print "Active node with uuid %s <%s>" % (node.uuid, node)
if _do_perform_action("destroy", actions):
target_name = options["hostname"]
for node in vm_launcher.list():
node_name = node.name
if node_name == target_name:
vm_launcher.destroy(node)
__invoke_plugin_actions(env, actions, "local_actions", [vm_launcher, options])
# Do we have remaining actions requiring an vm?
if len(actions) > 0:
print 'Setting up virtual machine'
vm_launcher.boot_and_connect()
_setup_vm(options, vm_launcher, actions)
def install_custom(p, automated=False, pkg_to_group=None):
"""Install a single custom package by name.
This method fetches names from custom.yaml that delegate to a method
in the custom/name.py program.
fab install_custom_package:package_name
"""
_setup_logging(env)
env.logger.info("Install custom software packages")
if not automated:
_parse_fabricrc()
_setup_edition(env)
_setup_distribution_environment()
_create_local_paths()
pkg_config = os.path.join(env.config_dir, "custom.yaml")
packages, pkg_to_group = _yaml_to_packages(pkg_config, None)
try:
env.logger.debug("Import %s" % p)
mod = __import__("cloudbio.custom.%s" % pkg_to_group[p],
fromlist=["cloudbio", "custom"])
except ImportError:
raise ImportError("Need to write a %s module in custom." %
pkg_to_group[p])
replace_chars = ["-"]
try:
for to_replace in replace_chars:
p = p.replace(to_replace, "_")
fn = getattr(mod, "install_%s" % p)
except AttributeError:
raise ImportError("Need to write a install_%s function in custom.%s" %
(p, pkg_to_group[p]))
fn(env)
def install_libraries(language):
"""High level target to install libraries for a specific language.
"""
_setup_logging(env)
_check_fabric_version()
_configure_fabric_environment(env, ignore_distcheck=True)
_do_library_installs(["%s-libs" % language])
def install_biolinux(target=None, flavor=None):
"""Main entry point for installing BioLinux on a remote server.
`flavor` allows customization of CloudBioLinux behavior. It can either
be a flavor name that maps to a corresponding directory in contrib/flavor
or the path to a custom directory. This can contain:
- alternative package lists (main.yaml, packages.yaml, custom.yaml)
- custom python code (nameflavor.py) that hooks into the build machinery
`target` allows running only particular parts of the build process. Valid choices are:
- packages Install distro packages
- custom Install custom packages
- chef_recipes Provision chef recipes
- libraries Install programming language libraries
- post_install Setup CloudMan, FreeNX and other system services
- cleanup Remove downloaded files and prepare images for AMI builds
"""
_setup_logging(env)
time_start = _print_time_stats("Config", "start")
_check_fabric_version()
_configure_fabric_environment(
env,
flavor,
ignore_distcheck=(target is not None
and target in ["libraries", "custom"]))
env.logger.debug("Target is '%s'" % target)
_perform_install(target, flavor)
_print_time_stats("Config", "end", time_start)
def install_libraries(language):
"""High level target to install libraries for a specific language.
"""
_setup_logging(env)
_check_fabric_version()
_configure_fabric_environment(env, ignore_distcheck=True)
_do_library_installs(["%s-libs" % language])
def install_custom(p, automated=False, pkg_to_group=None):
"""Install a single custom package by name.
This method fetches names from custom.yaml that delegate to a method
in the custom/name.py program.
fab install_custom_package:package_name
"""
_setup_logging(env)
env.logger.info("Install custom software packages")
if not automated:
_parse_fabricrc()
_setup_edition(env)
_setup_distribution_environment()
_create_local_paths()
pkg_config = os.path.join(env.config_dir, "custom.yaml")
packages, pkg_to_group = _yaml_to_packages(pkg_config, None)
try:
env.logger.debug("Import %s" % p)
mod = __import__("cloudbio.custom.%s" % pkg_to_group[p],
fromlist=["cloudbio", "custom"])
except ImportError:
raise ImportError("Need to write a %s module in custom." %
pkg_to_group[p])
replace_chars = ["-"]
try:
for to_replace in replace_chars:
p = p.replace(to_replace, "_")
fn = getattr(mod, "install_%s" % p)
except AttributeError:
raise ImportError("Need to write a install_%s function in custom.%s"
% (p, pkg_to_group[p]))
fn(env)
def _setup_cloudbiolinux(options):
def fabricrc_loader(env):
_setup_cloudbiolinux_fabric_properties(env, options)
flavor = get_main_options_string(options, "cloudbiolinux_flavor", None)
_setup_logging(env)
_configure_fabric_environment(env, flavor, fabricrc_loader=fabricrc_loader)
def _setup_cloudbiolinux(options):
def fabricrc_loader(env):
_setup_cloudbiolinux_fabric_properties(env, options)
flavor = get_main_options_string(options, "flavor",
DEFAULT_CLOUDBIOLINUX_FLAVOR)
_setup_logging(env)
_configure_fabric_environment(env, flavor, fabricrc_loader=fabricrc_loader)
def install_conda(p=None, flavor=None, automated=False):
if not automated:
_setup_logging(env)
_configure_fabric_environment(env, flavor, ignore_distcheck=True)
if p is not None:
conda.install_packages(env, packages=[p])
else:
pkg_install = _read_main_config()[0]
conda.install_packages(env, to_install=pkg_install)
def install_conda(p=None, flavor=None, automated=False):
if not automated:
_setup_logging(env)
_configure_fabric_environment(env, flavor, ignore_distcheck=True)
if p is not None:
conda.install_packages(env, packages=[p])
else:
pkg_install = _read_main_config()[0]
conda.install_packages(env, to_install=pkg_install)
def install_biolinux(target=None, packagelist=None, flavor=None):
"""Main entry point for installing Biolinux on a remote server.
This allows a different main package list (the main YAML file is passed in),
and/or use of Flavor. So you can say:
install_biolinux:packagelist=contrib/mylist/main.yaml,flavor=specialflavor
Both packagelist and flavor, as well as the Edition, can also be passed in
through the fabricrc file.
target can also be supplied on the fab CLI. Special targets are:
- packages Install distro packages
- custom Install custom packages
- libraries Install programming language libraries
- finalize Setup freenx and clean-up environment
"""
_setup_logging(env)
_check_fabric_version()
_parse_fabricrc()
_setup_edition(env)
_setup_flavor(flavor)
_setup_distribution_environment(
) # get parameters for distro, packages etc.
_create_local_paths()
env.logger.info("packagelist=%s" % packagelist)
pkg_install, lib_install = _read_main_config(packagelist) # read yaml
env.logger.info("Target=%s" % target)
if target is None or target == "packages":
if env.distribution in ["debian", "ubuntu"]:
_setup_apt_sources()
_setup_apt_automation()
_add_apt_gpg_keys()
_apt_packages(pkg_install)
elif env.distibution in ["centos"]:
_setup_yum_sources()
_yum_packages(pkg_install)
_setup_yum_bashrc()
else:
raise NotImplementedError("Unknown target distribution")
_update_biolinux_log(env, target, flavor)
if target is None or target == "custom":
_custom_installs(pkg_install)
if target is None or target == "libraries":
_do_library_installs(lib_install)
if target is None or target == "finalize":
env.edition.post_install()
env.flavor.post_install()
_cleanup_space()
if env.has_key("is_ec2_image") and env.is_ec2_image.upper() in [
"TRUE", "YES"
]:
_freenx_scripts()
_configure_cloudman(env)
_cleanup_ec2(env)
def install_libraries(language):
"""High level target to install libraries for a specific language.
"""
_setup_logging(env)
_check_fabric_version()
_parse_fabricrc()
_setup_edition(env)
_setup_flavor(None)
_setup_distribution_environment()
_create_local_paths()
_do_library_installs(["%s-libs" % language])
def install_libraries(language):
"""High level target to install libraries for a specific language.
"""
_setup_logging(env)
_check_fabric_version()
_parse_fabricrc()
_setup_edition(env)
_setup_flavor(None)
_setup_distribution_environment()
_create_local_paths()
_do_library_installs(["%s-libs" % language])
def install_brew(p=None, flavor=None):
"""Top level access to homebrew/linuxbrew packages.
p is a package name to install, or all configured packages if not specified.
"""
_setup_logging(env)
_configure_fabric_environment(env, flavor, ignore_distcheck=True)
system.install_homebrew(env)
if p is not None:
brew.install_packages(env, packages=[p])
else:
pkg_install = _read_main_config()[0]
brew.install_packages(env, to_install=pkg_install)
def install_biolinux(target=None, packagelist=None, flavor=None):
"""Main entry point for installing Biolinux on a remote server.
This allows a different main package list (the main YAML file is passed in),
and/or use of Flavor. So you can say:
install_biolinux:packagelist=contrib/mylist/main.yaml,flavor=specialflavor
Both packagelist and flavor, as well as the Edition, can also be passed in
through the fabricrc file.
target can also be supplied on the fab CLI. Special targets are:
- packages Install distro packages
- custom Install custom packages
- libraries Install programming language libraries
- finalize Setup freenx and clean-up environment
"""
_setup_logging(env)
_check_fabric_version()
_parse_fabricrc()
_setup_edition(env)
_setup_flavor(flavor)
_setup_distribution_environment() # get parameters for distro, packages etc.
_create_local_paths()
env.logger.info("packagelist=%s" % packagelist)
pkg_install, lib_install = _read_main_config(packagelist) # read yaml
env.logger.info("Target=%s" % target)
if target is None or target == "packages":
if env.distribution in ["debian", "ubuntu"]:
_setup_apt_sources()
_setup_apt_automation()
_add_apt_gpg_keys()
_apt_packages(pkg_install)
elif env.distibution in ["centos"]:
_setup_yum_sources()
_yum_packages(pkg_install)
_setup_yum_bashrc()
else:
raise NotImplementedError("Unknown target distribution")
_update_biolinux_log(env, target, flavor)
if target is None or target == "custom":
_custom_installs(pkg_install)
if target is None or target == "libraries":
_do_library_installs(lib_install)
if target is None or target == "finalize":
env.edition.post_install()
env.flavor.post_install()
_cleanup_space()
if env.has_key("is_ec2_image") and env.is_ec2_image.upper() in ["TRUE", "YES"]:
_freenx_scripts()
_configure_cloudman(env)
_cleanup_ec2(env)
def install_custom(p, automated=False, pkg_to_group=None, flavor=None):
"""
Install a single custom program or package by name.
This method fetches program name from ``config/custom.yaml`` and delegates
to a method in ``custom/*name*.py`` to proceed with the installation.
Alternatively, if a program install method is defined in the appropriate
package, it will be called directly (see param ``p``).
Usage: fab [-i key] [-u user] -H host install_custom:program_name
:type p: string
:param p: A name of the custom program to install. This has to be either a name
that is listed in ``custom.yaml`` as a subordinate to a group name or a
program name whose install method is defined in either ``cloudbio`` or
``custom`` packages
(e.g., ``cloudbio/custom/cloudman.py -> install_cloudman``).
:type automated: bool
:param automated: If set to True, the environment is not loaded and reading of
the ``custom.yaml`` is skipped.
"""
_setup_logging(env)
p = p.lower() # All packages listed in custom.yaml are in lower case
time_start = _print_time_stats("Custom install for '{0}'".format(p),
"start")
if not automated:
_configure_fabric_environment(env, flavor, ignore_distcheck=True)
pkg_config = get_config_file(env, "custom.yaml").base
packages, pkg_to_group = _yaml_to_packages(pkg_config, None)
try:
env.logger.debug("Import %s" % p)
# Allow direct calling of a program install method, even if the program
# is not listed in the custom list (ie, not contained as a key value in
# pkg_to_group). For an example, see 'install_cloudman' or use p=cloudman.
mod_name = pkg_to_group[p] if p in pkg_to_group else p
mod = __import__("cloudbio.custom.%s" % mod_name,
fromlist=["cloudbio", "custom"])
except ImportError:
raise ImportError("Need to write a %s module in custom." %
pkg_to_group[p])
replace_chars = ["-"]
try:
for to_replace in replace_chars:
p = p.replace(to_replace, "_")
fn = getattr(mod, "install_%s" % p)
except AttributeError:
raise ImportError("Need to write a install_%s function in custom.%s" %
(p, pkg_to_group[p]))
fn(env)
_print_time_stats("Custom install for '%s'" % p, "end", time_start)
def install_custom(p, automated=False, pkg_to_group=None, flavor=None):
"""
Install a single custom program or package by name.
This method fetches program name from ``config/custom.yaml`` and delegates
to a method in ``custom/*name*.py`` to proceed with the installation.
Alternatively, if a program install method is defined in the appropriate
package, it will be called directly (see param ``p``).
Usage: fab [-i key] [-u user] -H host install_custom:program_name
:type p: string
:param p: A name of the custom program to install. This has to be either a name
that is listed in ``custom.yaml`` as a subordinate to a group name or a
program name whose install method is defined in either ``cloudbio`` or
``custom`` packages
(e.g., ``cloudbio/custom/cloudman.py -> install_cloudman``).
:type automated: bool
:param automated: If set to True, the environment is not loaded and reading of
the ``custom.yaml`` is skipped.
"""
_setup_logging(env)
p = p.lower() # All packages listed in custom.yaml are in lower case
time_start = _print_time_stats("Custom install for '{0}'".format(p), "start")
if not automated:
_configure_fabric_environment(env, flavor)
pkg_config = get_config_file(env, "custom.yaml").base
packages, pkg_to_group = _yaml_to_packages(pkg_config, None)
try:
env.logger.debug("Import %s" % p)
# Allow direct calling of a program install method, even if the program
# is not listed in the custom list (ie, not contained as a key value in
# pkg_to_group). For an example, see 'install_cloudman' or use p=cloudman.
mod_name = pkg_to_group[p] if p in pkg_to_group else p
mod = __import__("cloudbio.custom.%s" % mod_name,
fromlist=["cloudbio", "custom"])
except ImportError:
raise ImportError("Need to write a %s module in custom." %
pkg_to_group[p])
replace_chars = ["-"]
try:
for to_replace in replace_chars:
p = p.replace(to_replace, "_")
fn = getattr(mod, "install_%s" % p)
except AttributeError:
raise ImportError("Need to write a install_%s function in custom.%s"
% (p, pkg_to_group[p]))
fn(env)
_print_time_stats("Custom install for '%s'" % p, "end", time_start)
def install_brew(p=None, version=None, flavor=None, automated=False):
"""Top level access to homebrew/linuxbrew packages.
p is a package name to install, or all configured packages if not specified.
"""
if not automated:
_setup_logging(env)
_configure_fabric_environment(env, flavor, ignore_distcheck=True)
if p is not None:
if version:
p = "%s==%s" % (p, version)
brew.install_packages(env, packages=[p])
else:
pkg_install = _read_main_config()[0]
brew.install_packages(env, to_install=pkg_install)
def install_brew(p=None, version=None, flavor=None, automated=False):
"""Top level access to homebrew/linuxbrew packages.
p is a package name to install, or all configured packages if not specified.
"""
if not automated:
_setup_logging(env)
_configure_fabric_environment(env, flavor, ignore_distcheck=True)
if p is not None:
if version:
p = "%s==%s" % (p, version)
brew.install_packages(env, packages=[p])
else:
pkg_install = _read_main_config()[0]
brew.install_packages(env, to_install=pkg_install)
def install_custom(p, automated=False, pkg_to_group=None):
"""Install a single custom package by name.
This method fetches names from custom.yaml that delegate to a method
in the custom/name.py program. Alternatively, if a program install method is
defined in approapriate package, it will be called directly (see param p).
Usage: fab [-i key] [-u user] -H host install_custom:program_name
:type p: string
:param p: A name of a custom program to install. This has to be either a name
that is listed in custom.yaml as a subordinate to a group name or a
program name whose install method is defined in either cloudbio or
custom packages (eg, install_cloudman).
:type automated: bool
:param automated: If set to True, the environment is not loaded and reading of
the custom.yaml is skipped.
"""
_setup_logging(env)
p = p.lower() # All packages are listed in custom.yaml are in lower case
time_start = _print_time_stats("Custom install for '{0}'".format(p), "start")
if not automated:
_parse_fabricrc()
_setup_edition(env)
_setup_distribution_environment()
_create_local_paths()
pkg_config = os.path.join(env.config_dir, "custom.yaml")
packages, pkg_to_group = _yaml_to_packages(pkg_config, None)
try:
env.logger.debug("Import %s" % p)
# Allow direct calling of a program install method, even if the program
# is not listed in the custom list (ie, not contained as a key value in
# pkg_to_group). For an example, see 'install_cloudman' or use p=cloudman.
mod_name = pkg_to_group[p] if p in pkg_to_group else p
mod = __import__("cloudbio.custom.%s" % mod_name,
fromlist=["cloudbio", "custom"])
except ImportError:
raise ImportError("Need to write a %s module in custom." %
pkg_to_group[p])
replace_chars = ["-"]
try:
for to_replace in replace_chars:
p = p.replace(to_replace, "_")
fn = getattr(mod, "install_%s" % p)
except AttributeError:
raise ImportError("Need to write a install_%s function in custom.%s"
% (p, pkg_to_group[p]))
fn(env)
_print_time_stats("Custom install for '%s'" % p, "end", time_start)
def install_puppet_class(classes, automated=False, flavor=None):
"""Install one or more puppet classes by name.
Usage: fab [-i key] [-u user] -H host install_puppet_class:class
:type classes: string or list
:param classes: TODO
:type automated: bool
:param automated: If set to True, the environment is not loaded.
"""
_setup_logging(env)
if not automated:
_configure_fabric_environment(env, flavor)
time_start = _print_time_stats("Puppet provision for class(es) '{0}'".format(classes), "start")
classes = classes if isinstance(classes, list) else [classes]
_puppet_provision(env, classes)
_print_time_stats("Puppet provision for classes(s) '%s'" % classes, "end", time_start)
def install_puppet_class(classes, automated=False, flavor=None):
"""Install one or more puppet classes by name.
Usage: fab [-i key] [-u user] -H host install_puppet_class:class
:type classes: string or list
:param classes: TODO
:type automated: bool
:param automated: If set to True, the environment is not loaded.
"""
_setup_logging(env)
if not automated:
_configure_fabric_environment(env, flavor)
time_start = _print_time_stats("Puppet provision for class(es) '{0}'".format(classes), "start")
classes = classes if isinstance(classes, list) else [classes]
_puppet_provision(env, classes)
_print_time_stats("Puppet provision for classes(s) '%s'" % classes, "end", time_start)
def install_biolinux(target=None, flavor=None):
"""Main entry point for installing BioLinux on a remote server.
`flavor` allows customization of CloudBioLinux behavior. It can either
be a flavor name that maps to a corresponding directory in contrib/flavor
or the path to a custom directory. This can contain:
- alternative package lists (main.yaml, packages.yaml, custom.yaml)
- custom python code (nameflavor.py) that hooks into the build machinery
`target` allows running only particular parts of the build process. Valid choices are:
- packages Install distro packages
- custom Install custom packages
- libraries Install programming language libraries
- post_install Setup CloudMan, FreeNX and other system services
- cleanup Remove downloaded files and prepare images for AMI builds
"""
_setup_logging(env)
time_start = _print_time_stats("Config", "start")
_check_fabric_version()
_configure_fabric_environment(env, flavor)
env.logger.debug("Target is '%s'" % target)
pkg_install, lib_install, custom_ignore = _read_main_config()
if target is None or target == "packages":
_configure_and_install_native_packages(env, pkg_install)
if env.nixpkgs: # ./doc/nixpkgs.md
_setup_nix_sources()
_nix_packages(pkg_install)
_update_biolinux_log(env, target, flavor)
if target is None or target == "custom":
_custom_installs(pkg_install, custom_ignore)
if target is None or target == "libraries":
_do_library_installs(lib_install)
if target is None or target == "post_install":
env.edition.post_install(pkg_install=pkg_install)
env.flavor.post_install()
if target is None or target == "cleanup":
_cleanup_space(env)
if env.has_key("is_ec2_image") and env.is_ec2_image.upper() in ["TRUE", "YES"]:
_cleanup_ec2(env)
_print_time_stats("Config", "end", time_start)
def _setup_environment(config):
env.distribution = config["distribution"]
env.dist_name = config.get("dist_name", None)
env.system_install = config["base_install"]
_setup_logging(env)
distribution._setup_distribution_environment()
result = run("echo $HOSTNAME")
config["host"] = result.strip()
run("chmod a+rx $HOME")
shell_config = None
for test in [".bash_profile", ".bashrc"]:
if exists(test):
shell_config = test
break
assert shell_config is not None
run("chmod a+r %s" % shell_config)
if contains(shell_config, "^export PATH$"):
comment(shell_config, "^export PATH$")
config["shell_config"] = shell_config
def install_chef_recipe(recipe, automated=False, flavor=None):
"""Install one or more chef recipes by name.
Usage: fab [-i key] [-u user] -H host install_chef_recipe:recipe
:type recipe: string or list
:param recipe: TODO
:type automated: bool
:param automated: If set to True, the environment is not loaded.
"""
_setup_logging(env)
if not automated:
_configure_fabric_environment(env, flavor)
time_start = _print_time_stats("Chef provision for recipe(s) '{0}'".format(recipe), "start")
_configure_chef(env, chef)
recipes = recipe if isinstance(recipe, list) else [recipe]
for recipe_to_add in recipes:
chef.add_recipe(recipe_to_add)
_chef_provision(env, recipes)
_print_time_stats("Chef provision for recipe(s) '%s'" % recipe, "end", time_start)
def install_chef_recipe(recipe, automated=False, flavor=None):
"""Install one or more chef recipes by name.
Usage: fab [-i key] [-u user] -H host install_chef_recipe:recipe
:type recipe: string or list
:param recipe: TODO
:type automated: bool
:param automated: If set to True, the environment is not loaded.
"""
_setup_logging(env)
if not automated:
_configure_fabric_environment(env, flavor)
time_start = _print_time_stats("Chef provision for recipe(s) '{0}'".format(recipe), "start")
_configure_chef(env, chef)
recipes = recipe if isinstance(recipe, list) else [recipe]
for recipe_to_add in recipes:
chef.add_recipe(recipe_to_add)
_chef_provision(env, recipes)
_print_time_stats("Chef provision for recipe(s) '%s'" % recipe, "end", time_start)
def install_custom(p, automated=False, pkg_to_group=None, flavor=None):
"""
Install a single custom program or package by name.
This method fetches program name from ``config/custom.yaml`` and delegates
to a method in ``custom/*name*.py`` to proceed with the installation.
Alternatively, if a program install method is defined in the appropriate
package, it will be called directly (see param ``p``).
Usage: fab [-i key] [-u user] -H host install_custom:program_name
:type p: string
:param p: A name of the custom program to install. This has to be either a name
that is listed in ``custom.yaml`` as a subordinate to a group name or a
program name whose install method is defined in either ``cloudbio`` or
``custom`` packages
(e.g., ``cloudbio/custom/cloudman.py -> install_cloudman``).
:type automated: bool
:param automated: If set to True, the environment is not loaded and reading of
the ``custom.yaml`` is skipped.
"""
p = p.lower() # All packages listed in custom.yaml are in lower case
if not automated:
_setup_logging(env)
_configure_fabric_environment(env, flavor, ignore_distcheck=True)
pkg_config = get_config_file(env, "custom.yaml").base
packages, pkg_to_group = _yaml_to_packages(pkg_config, None)
time_start = _print_time_stats("Custom install for '{0}'".format(p),
"start")
fn = _custom_install_function(env, p, pkg_to_group)
fn(env)
## TODO: Replace the previous 4 lines with the following one, barring
## objections. Slightly different behavior because pkg_to_group will be
## loaded regardless of automated if it is None, but IMO this shouldn't
## matter because the following steps look like they would fail if
## automated is True and pkg_to_group is None.
# _install_custom(p, pkg_to_group)
_print_time_stats("Custom install for '%s'" % p, "end", time_start)
def install_custom(p, automated=False, pkg_to_group=None, flavor=None):
"""
Install a single custom program or package by name.
This method fetches program name from ``config/custom.yaml`` and delegates
to a method in ``custom/*name*.py`` to proceed with the installation.
Alternatively, if a program install method is defined in the appropriate
package, it will be called directly (see param ``p``).
Usage: fab [-i key] [-u user] -H host install_custom:program_name
:type p: string
:param p: A name of the custom program to install. This has to be either a name
that is listed in ``custom.yaml`` as a subordinate to a group name or a
program name whose install method is defined in either ``cloudbio`` or
``custom`` packages
(e.g., ``cloudbio/custom/cloudman.py -> install_cloudman``).
:type automated: bool
:param automated: If set to True, the environment is not loaded and reading of
the ``custom.yaml`` is skipped.
"""
p = p.lower() # All packages listed in custom.yaml are in lower case
if not automated:
_setup_logging(env)
_configure_fabric_environment(env, flavor, ignore_distcheck=True)
pkg_config = get_config_file(env, "custom.yaml").base
packages, pkg_to_group = _yaml_to_packages(pkg_config, None)
time_start = _print_time_stats("Custom install for '{0}'".format(p), "start")
fn = _custom_install_function(env, p, pkg_to_group)
fn(env)
## TODO: Replace the previous 4 lines with the following one, barring
## objections. Slightly different behavior because pkg_to_group will be
## loaded regardless of automated if it is None, but IMO this shouldn't
## matter because the following steps look like they would fail if
## automated is True and pkg_to_group is None.
# _install_custom(p, pkg_to_group)
_print_time_stats("Custom install for '%s'" % p, "end", time_start)
def setup_environment():
"""Setup environment with required data file locations.
"""
_setup_logging(env)
_add_defaults()
_setup_distribution_environment()
def setup_environment():
"""Setup environment with required data file locations.
"""
_setup_logging(env)
_add_defaults()
_configure_fabric_environment(env, ignore_distcheck=True)
def install_biolinux(target=None, packagelist=None, flavor=None, environment=None):
"""Main entry point for installing Biolinux on a remote server.
This allows a different main package list (the main YAML file is passed in),
and/or use of Flavor. So you can say:
install_biolinux:packagelist=contrib/mylist/main.yaml,flavor=specialflavor
Both packagelist and flavor, as well as the Edition, can also be passed in
through the fabricrc file.
target can also be supplied on the fab CLI. Special targets are:
- packages Install distro packages
- custom Install custom packages
- libraries Install programming language libraries
- post_install Setup CloudMan, FreeNX and other system services
- cleanup Remove downloaded files and prepare images for AMI builds
environment allows adding additional information on the command line -
usually for defining environments, for example environment=testing, or
environment=production, will set the deployment environment and tune
post-installation settings.
"""
_setup_logging(env)
time_start = _print_time_stats("Config", "start")
_check_fabric_version()
_parse_fabricrc()
_setup_edition(env)
_setup_flavor(flavor, environment)
_setup_distribution_environment() # get parameters for distro, packages etc.
_create_local_paths()
env.logger.info("packagelist=%s" % packagelist)
pkg_install, lib_install = _read_main_config(packagelist) # read yaml
env.logger.info("Target=%s" % target)
if target is None or target == "packages":
if env.distribution in ["debian", "ubuntu"]:
_setup_apt_sources()
_setup_apt_automation()
_add_apt_gpg_keys()
_apt_packages(pkg_install)
elif env.distribution in ["centos", "scientificlinux"]:
_setup_yum_sources()
_yum_packages(pkg_install)
_setup_yum_bashrc()
else:
raise NotImplementedError("Unknown target distribution")
if env.nixpkgs: # ./doc/nixpkgs.md
_setup_nix_sources()
_nix_packages(pkg_install)
_update_biolinux_log(env, target, flavor)
if target is None or target == "custom":
_custom_installs(pkg_install)
if target is None or target == "libraries":
_do_library_installs(lib_install)
if target is None or target == "post_install":
env.edition.post_install()
env.flavor.post_install()
if target is None or target == "cleanup":
_cleanup_space(env)
if env.has_key("is_ec2_image") and env.is_ec2_image.upper() in ["TRUE", "YES"]:
_cleanup_ec2(env)
_print_time_stats("Config", "end", time_start)
def setup_environment():
"""Setup environment with required data file locations.
"""
_setup_logging(env)
_add_defaults()
_configure_fabric_environment(env, ignore_distcheck=True)
def setup_environment():
"""Setup environment with required data file locations.
"""
_setup_logging(env)
_add_defaults()
_setup_distribution_environment()
def install_biolinux(target=None, packagelist=None, flavor=None, environment=None,
pkg_config_file_path=None):
"""
Main entry point for installing BioLinux on a remote server.
``packagelist`` should point to a top level file (eg, ``main.yaml``) listing
all the package categories that should be installed. This allows a different
package list and/or use of Flavor. So you can say::
install_biolinux:packagelist=contrib/mylist/main.yaml,flavor=specialflavor
``pkg_config_file_path`` can be used to specify a path where a custom
``packages.yaml`` and ``packages-[dist].yaml`` are located, allowing fine-
grained (i.e., individual package) customization. Otherwise, default
to ``./contrib`` where the CBL files are defined.
Both ``packagelist`` and ``flavor``, as well as the Edition, can also be
passed in through the ``fabricrc`` file.
target can also be supplied on the fab CLI. Special targets are:
- packages Install distro packages
- custom Install custom packages
- libraries Install programming language libraries
- post_install Setup CloudMan, FreeNX and other system services
- cleanup Remove downloaded files and prepare images for AMI builds
``environment`` allows adding additional information on the command line -
usually for defining environments, for example ``environment=testing``, or
``environment=production``, will set the deployment environment and tune
post-installation settings.
"""
_setup_logging(env)
time_start = _print_time_stats("Config", "start")
_check_fabric_version()
_parse_fabricrc()
_setup_edition(env)
_setup_flavor(flavor, environment)
_setup_distribution_environment() # get parameters for distro, packages etc.
_create_local_paths()
env.logger.debug("Meta-package list is '%s'" % packagelist)
env.logger.debug("File path for explicit packages is '%s'" % pkg_config_file_path)
env.logger.debug("Target is '%s'" % target)
pkg_install, lib_install, custom_ignore = _read_main_config(packagelist) # read main yaml
if target is None or target == "packages":
if env.distribution in ["debian", "ubuntu"]:
_setup_apt_sources()
_setup_apt_automation()
_add_apt_gpg_keys()
_apt_packages(pkg_install, pkg_config_file_path=pkg_config_file_path)
elif env.distribution in ["centos", "scientificlinux"]:
_setup_yum_sources()
_yum_packages(pkg_install)
_setup_yum_bashrc()
else:
raise NotImplementedError("Unknown target distribution")
if env.nixpkgs: # ./doc/nixpkgs.md
_setup_nix_sources()
_nix_packages(pkg_install)
_update_biolinux_log(env, target, flavor)
if target is None or target == "custom":
_custom_installs(pkg_install, custom_ignore)
if target is None or target == "libraries":
_do_library_installs(lib_install)
if target is None or target == "post_install":
env.edition.post_install(pkg_install=pkg_install)
env.flavor.post_install()
if target is None or target == "cleanup":
_cleanup_space(env)
if env.has_key("is_ec2_image") and env.is_ec2_image.upper() in ["TRUE", "YES"]:
_cleanup_ec2(env)
_print_time_stats("Config", "end", time_start)
