def expand(self):
if self.is_properly_installed:
log.debug('{}-{} already installed'.format(self.name,
self.version))
return
if self.preinstall_callback:
self.preinstall_callback()
self.install_templates()
log.info('Extracting {}-{}'.format(self.name, self.version))
with self.tempdir() as temp, open(
os.devnull,
'w') as FNULL: # , zipfile.ZipFile(self.path) as zip_file:
# zip_file.extractall(temp)
# Python's zip file utility has been broken since 2012 im forced to come up with this work around because
# i absolutely need to maintain the file permisions; ie. executable/read
subprocess.call('unzip {source} -d {destination}'.format(
source=self.path, destination=temp, shell=True),
stdout=FNULL,
stderr=FNULL)
self.installer(self, temp, self.environment_vars)
if self.postinstall_callback:
self.postinstall_callback()
def expand(self):
if self.is_properly_installed:
log.debug('{}-{} already installed'.format(self.name,
self.version))
return
if self.preinstall_callback:
self.preinstall_callback()
if self.path.endswith('tar.xz'):
@contextlib.contextmanager
def xz_tar(path):
"""
Ensures all all the files are going to get closed properly
"""
with contextlib.closing(lzma.LZMAFile(path)) as xz:
yield tarfile.open(fileobj=xz)
tarfile_obj = xz_tar(self.path)
else:
tarfile_obj = tarfile.open(self.path, 'r:*')
self.install_templates()
log.info('Extracting {}-{}'.format(self.name, self.version))
with self.tempdir() as temp, tarfile_obj as tar_file:
tar_file.extractall(temp)
self.installer(self, temp, self.environment_vars)
if self.postinstall_callback:
self.postinstall_callback()
def download(self):
log.debug('Downloading {}-{}, saving it to {}'.format(
self.name, self.version, self.path))
lamnfyc.utils.download(self.url, self.path)
log.debug('Download {}-{} complete'.format(self.name, self.version))
# verify everything was downloaded correctly
return self.verify_cache(raise_exception=True)
def valid_signature(self):
if self.sha256_signature:
# if there is an sha256 signature to compare it to, do it!
known_signature = self.sha256_signature
hash_module = hashlib.sha256()
elif self.md5_signature:
# if there is an md5 signature to compare it to, do it!
known_signature = self.md5_signature
hash_module = hashlib.md5()
else:
log.debug('{}-{} did not have and signatures to compare'.format(
self.name, self.version))
return True
with open(self.path, "rb") as f:
for chunk in iter(lambda: f.read(16 * 1024), b""):
hash_module.update(chunk)
# check that the signatures match
generated_hash = hash_module.hexdigest()
signatures_matched = generated_hash == known_signature
if signatures_matched:
log.debug('{}-{} hash good'.format(self.name, self.version))
else:
log.warn(
'{}-{} hash bad. Received: {}, but is expected: {}'.format(
self.name, self.version, generated_hash, known_signature))
return signatures_matched
def check_version(package, installed_version):
installed_version = installed_version.replace('v', '')
if installed_version == package.version:
return True
log.debug('{} already exists, but version {} did not match {}'.format(package.installer.path, installed_version,
package.version))
return False
def tempdir(self):
temp_dir = tempfile.mkdtemp()
log.debug('New temp dir for {}-{} made: {}'.format(
self.name, self.version, temp_dir))
try:
yield temp_dir
finally:
shutil.rmtree(temp_dir)
log.debug('Temp dir for {}-{} deleted: {}'.format(
self.name, self.version, temp_dir))
def verify_cache(self, raise_exception=False):
if os.path.exists(self.path):
if self.valid_signature():
return True
MESSAGE = 'Removing the {}-{} cache file, signature was no good: {}'.format(
self.name, self.version, self.path)
log.debug(MESSAGE)
os.remove(self.path)
if raise_exception:
raise Exception(MESSAGE)
return False
def _main(args):
environment_config = lamnfyc.utils.Configuration(args.config)
# need the absolute path to the environment
lamnfyc.settings.environment_path = os.path.abspath(
os.path.join(os.path.abspath(os.path.curdir),
args.environment).rstrip('/'))
# sets up the system path local to where the yaml file is so you can import the pre/post hooks
sys.path.insert(0, os.path.dirname(os.path.abspath(args.config)))
# set the logging level to console only
log.handlers[0].setLevel(args.verbosity)
# create the cache dir if its missing
if not os.path.isdir(lamnfyc.settings.CACHE_PATH):
os.mkdir(lamnfyc.settings.CACHE_PATH)
if not args.reuse:
log.debug('Starting environment: {}'.format(
lamnfyc.settings.environment_path))
# error out if the environment already exists
if os.path.isdir(lamnfyc.settings.environment_path):
log.fatal('ERROR: File already exists and is not a directory.')
log.fatal(
'Please provide a different path or delete the directory.')
sys.exit(3)
# make sure all the paths exists
os.mkdir(lamnfyc.settings.environment_path)
# Standard unix installation structure
os.mkdir(os.path.join(lamnfyc.settings.environment_path, 'lib'))
os.mkdir(os.path.join(lamnfyc.settings.environment_path, 'bin'))
os.mkdir(os.path.join(lamnfyc.settings.environment_path, 'share'))
os.mkdir(os.path.join(lamnfyc.settings.environment_path, 'include'))
# Appended structure, to house configuration files, logs, and sock/run files
os.mkdir(os.path.join(lamnfyc.settings.environment_path, 'conf'))
os.mkdir(os.path.join(lamnfyc.settings.environment_path, 'logs'))
os.mkdir(os.path.join(lamnfyc.settings.environment_path, 'run'))
else:
log.warn('Reuse mode enabled, this is not fully supported')
# initialize the file level logging
start_file_log(lamnfyc.settings.environment_path)
if environment_config.preinstall_hook:
environment_config.preinstall_hook()
environment_config.prompt_missing(missing_only=not args.prompt_all)
kwargs = {
'environment_path': lamnfyc.settings.environment_path,
'enironment_variables': variable_order(environment_config.env),
'unset_variables': ' '.join(environment_config.env.keys())
}
path = os.path.join(lamnfyc.settings.BASE_PATH, 'templates')
files = [
os.path.join(root, file) for root, dir, files in os.walk(path)
for file in files
]
for file in files:
file_path = os.path.join(lamnfyc.settings.environment_path,
file.replace(path + os.path.sep, ''))
with open(file_path, 'w') as file_out:
file_out.write(jinja2.Template(open(file).read()).render(**kwargs))
# If it goes inside /bin then give it exec permissions
if file_path.replace(lamnfyc.settings.environment_path + os.path.sep,
'').split(os.path.sep)[0] == 'bin':
os.chmod(file_path, os.stat(file).st_mode | stat.S_IEXEC)
# after all the environment variables have been written, lets read them back up to get nice and clean values
# without any $VARIABLE in them
environment_config.reload_env(lamnfyc.settings.environment_path)
# generate all the packages we need to download
downloads = []
for package_item in environment_config.packages:
package = lamnfyc.utils.import_package(package_item['name'],
package_item['version'])
package.environment_vars = environment_config.env
downloads.append(package)
for subpackage in package.dependencies():
downloads.append(subpackage)
subpackage.environment_vars = environment_config.env
# download all the packages that are missing
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
futures = executor.map(
lambda package: package.download()
if not package.cache_exists else None, downloads)
for future in futures:
# if any of the futures threw an error it will raise them here and thus the program will halt
continue
# Install all packages, uppermost first, meaning;
# If say Package1 depends on Package2 which in turn that depends on Package3, the order or the install will be:
# Package3 gets installed first, then Package2, and lastly Package1
for package_item in environment_config.packages:
package = lamnfyc.utils.import_package(package_item['name'],
package_item['version'])
for subpackage in package.dependencies():
subpackage.expand()
package.expand()
if environment_config.postinstall_callback:
environment_config.postinstall_callback()