def find_candidate_yaml_files(user_places_to_look=None):
'''
Finds YAML files in the specified paths. Allows the user to pass custom directories.
'''
logger = log_util.logger_for_me(find_candidate_yaml_files)
# look in default places
dcluster_places_to_look = main_config.paths('profiles')
places_to_look = collection_util.defensive_copy(dcluster_places_to_look)
# also include user-specified places
if user_places_to_look is not None and isinstance(user_places_to_look, list):
places_to_look.extend(user_places_to_look)
logger.debug('places to look for profiles: {}'.format(places_to_look))
# convert '~' to user's home directory
places_to_look = [
os.path.expanduser(place_to_look)
for place_to_look
in places_to_look
]
# find all the yaml files, remember found place
yaml_files = OrderedDict()
for place_to_look in places_to_look:
with_yml = fs.find_files_with_extension(place_to_look, '.yml')
with_yaml = fs.find_files_with_extension(place_to_look, '.yaml')
yaml_files[place_to_look] = []
yaml_files[place_to_look].extend(with_yml)
yaml_files[place_to_look].extend(with_yaml)
return yaml_files
def ensure_docker_compose():
'''
Looks for docker_compose in PATH. If the executable is not found, it will try to download
it from the docker-compose repository. Exits normally if successful, raises ValueError if
the executable cannot be downloaded.
'''
log = logger.logger_for_me(ensure_docker_package)
# check for docker-compose in PATH
docker_compose_path = distutils.spawn.find_executable('docker-compose')
if not docker_compose_path:
# prepare to download it at user HOME
server_url = docker_compose_url()
output_dir = os.path.expanduser('~/.dcluster/bin')
fs.create_dir_dont_complain(output_dir) # ensure download dir
docker_compose_path = os.path.join(output_dir, 'docker-compose')
# download if this fails then give up
download.download_to_file_strict(server_url, docker_compose_path)
# make it executable
st = os.stat(docker_compose_path)
os.chmod(docker_compose_path, st.st_mode | stat.S_IEXEC)
# check for docker-compose version
cmd = 'docker-compose -v'
stdout, stderr, rc = runit.execute(cmd, env=os.environ)
if 'docker-compose' not in stdout or rc != 0:
err_msg = 'Not able to use docker-compose (rc={}):\n{}\n{}'
raise ValueError(err_msg.format(rc, stdout, stderr))
log.info('Found docker-compose at: {}'.format(docker_compose_path))
def install_with_pip(package_name):
'''
Calls pip install --user <package_name>.
'''
log = logger.logger_for_me(install_with_pip)
my_pip = detect_pip_version()
cmd = '{} install --user {}'.format(my_pip, package_name)
log.debug('Calling: >>{}<<'.format(cmd))
runit.execute(cmd, logger=log, log_level=logging.INFO)
def get_workpath(args):
'''
Set the work path of dcluster. By default, it is set by the configuration (paths:work),
but it can be overridden by the user using the --workpath optional variable.
'''
log = logger.logger_for_me(get_workpath)
if args.workpath is not None:
workpath = args.workpath
else:
# work_path_with_shell_variables = main_config.paths('work')
# log.debug('workpath shell %s' % work_path_with_shell_variables)
# workpath = fs_util.evaluate_shell_path(work_path_with_shell_variables)
workpath = main_config.paths('work')
# create the directory, it may not exist but we need it now
log.debug('workpath %s' % workpath)
fs_util.create_dir_dont_complain(workpath)
return workpath
def download_to_file(server_url, output_filename):
'''
Downloads the file at baseURL and saves the file
to output_filename. Overwrites the file.
Returns None if downloaded, error string if problem with download
'''
log = logger.logger_for_me(download_to_file)
# ignore bad certificates using monkey patch
ssl._create_default_https_context = ssl._create_unverified_context
# code is version dependent
python_version = sys.version_info[0]
if python_version < 3:
return download_to_file_py2(server_url, output_filename, log)
else:
return download_to_file_py3(server_url, output_filename, log)
def ensure_docker_package():
'''
Tries to import docker module. If the module is not found, it will try to install docker via
pip. Exits normally if successful, raises ImportError if the import fails.
'''
log = logger.logger_for_me(ensure_docker_package)
# keep track of docker installation via pip function:
# if we install docker right now, "import docker" fails
just_installed = False
if not pip_user.is_package_installed('docker'):
# try to install it via pip
pip_user.install_with_pip('docker')
just_installed = True
if not just_installed:
import docker
log.info('Python API for docker found: {}'.format(docker.__version__))
def get_all_available_profiles(user_places_to_look):
'''
Finds the YAML files with cluster information.
The YAML files *must* have one of the following items:
- it is a dictionary of exactly one key/value pair
- the value is a dictionary, the key will be the profile identifier
- either the items 'cluster_type' OR 'extend' should be in the internal dictionary
(having both is allowed)
TODO
- enforce restrictions?
'''
logger = log_util.logger_for_me(get_all_available_profiles)
candidate_yaml_files = find_candidate_yaml_files(user_places_to_look)
# here we overwrite any repeated yaml file without consideration of where they were found
profile_yaml_files = []
for location, candidates in candidate_yaml_files.items():
for candidate in candidates:
profile_yaml_path = os.path.join(location, candidate)
profile_yaml_files.append(profile_yaml_path)
available_profiles = OrderedDict()
for profile_file in profile_yaml_files:
with open(profile_file, 'r') as ff:
yaml_dict = yaml.load(ff, Loader=yaml.SafeLoader)
# inform the user if a profile is to be updated
for profile_name in yaml_dict.keys():
if profile_name in available_profiles:
log_msg = 'profile \"{}\" will be updated with definition at: {}'
logger.info(log_msg.format(profile_name, profile_file))
# a YAML file may have many profiles, this will add all of them
# allow override of previously defined profiles
available_profiles.update(yaml_dict)
logger.debug('Found profiles: {}'.format(list(available_profiles.keys())))
return available_profiles
def find_playbook_path(playbook_name):
'''
Finds the path for a dansible playbook given a name.
The path is one of the following:
1. a subdirectory of the current user directory, i.e. $PWD/<playbook_name>
2. a subdirectory of the 'installed' dansible playbooks, e.g. $HOME/.dcluster/dansible/<playbook_name>
3. a subdirectory of the 'default' dansible playbooks, e.g.
/usr/share/dcluster/ansible_static/<playbook_name>
These paths are in order of precedence: the first subdirectory is chosen if it exists,
if not try the second and so on.
The path *must* contain the playbook as "playbook.yml" (not checked here)
Raises ValueError if no directory containing the playbook_name is found.
'''
log = logger.logger_for_me(find_playbook_path)
# the three paths that may contain the subdirectory
places_to_look = [
os.getcwd(),
installed_playbook_path(),
default_playbook_path()
]
found_playbook_path = None
for place_to_look in places_to_look:
possible_path = os.path.join(place_to_look, playbook_name)
if os.path.isdir(possible_path):
found_playbook_path = possible_path
break
if not found_playbook_path:
raise ValueError(
'Could not find {} directory in any of these places: {}'.format(
playbook_name, places_to_look))
log.debug('Found playbook path: {}'.format(found_playbook_path))
return found_playbook_path