def check_install(software, quiet=True):
'''attempt to run the command for some software, and
return True if installed. The command line utils will not run
without this check.
Parameters
==========
software: the software to run.
command: the command to run. If not provided, will use --version
quiet: if True, don't print verbose output
'''
if command is None:
command = '--version'
cmd = [software, command]
try:
version = run_command(cmd,software)
except: # FileNotFoundError
return False
if version is not None:
if quiet is False and version['return_code'] == 0:
version = version['message']
bot.info("Found %s version %s" % (software.upper(), version))
return True
return False
def load_filesystem(base, quiet=False):
'''load a filesystem based on a root path, which is usually /scif
Parameters
==========
base: base to load.
Returns
=======
config: a parsed recipe configuration for SCIF
'''
from scif.defaults import SCIF_APPS
if os.path.exists(SCIF_APPS):
apps = os.listdir(SCIF_APPS)
config = {'apps': {}}
for app in apps:
path = '%s/%s/scif/%s.scif' % (SCIF_APPS, app, app)
if os.path.exists(path):
recipe = load_recipe(path)
config['apps'][app] = recipe['apps'][app]
if len(config['apps']) > 0:
if quiet is False:
bot.info('Found configurations for %s scif apps' %
len(config['apps']))
bot.info('\n'.join(list(config['apps'].keys())))
return config
def install_commands(self, app, settings, config):
"""install will finally, issue commands to install the app.
Parameters
==========
app should be the name of the app, for lookup in config['apps']
settings: the output of _init_app(), a dictionary of environment vars
config: should be the config for the app obtained with self.app(app)
"""
if "appinstall" in config:
# Change directory so the APP is $PWD
pwd = os.getcwd()
os.chdir(settings["approot"])
# Set strict mode to ensure exit on error
command = ["set -e"] + config["appinstall"]
# issue install commands
cmd = "\n".join(command)
bot.info("+ " + "appinstall ".ljust(5) + app)
retval = os.system(cmd)
if retval != 0:
bot.exit("Return value %s for install of %s" % (retval, app))
# Go back to previous location
os.chdir(pwd)
def test(self, app=None, args=None):
"""test an app. This means the following:
1. Check that the app is valid for the client. Don't proceed otherwise
2. Set the client app to be active
3. update the environment to indicate the app is active
4. set the entry point and entry folder relevant to the app
5. Run interactively, and return the entry code to the client
Parameters
==========
app: the name of the scif app to run. If none provided, all apps are tested.
args: a list of one or more additional arguments to pass to runscript
"""
# Does the application have a test script?
if app in self.apps():
self.activate(app, args=args)
if not self._set_entrypoint(app, "SCIF_APPTEST", args):
bot.info("No tests defined for this app.")
sys.exit(1)
return self._exec(app, interactive=True, exit=True)
def install_files(self, app, settings, config):
'''install files will add files (or directories) to a destination.
If none specified, they are placed in the app base
Parameters
==========
app should be the name of the app, for lookup in config['apps']
settings: the output of _init_app(), a dictionary of environment vars
config: should be the config for the app obtained with self.app(app)
'''
if "appfiles" in config:
files = config['appfiles']
bot.info('+ ' + 'appfiles '.ljust(5) + app)
for pair in files:
# Step 1: determine source and destination
src, dest = get_parts(pair, default=settings['approot'])
# Step 2: copy source to destination
cmd = ['cp']
if os.path.isdir(src):
cmd.append('-R')
elif os.path.exists(src):
cmd = cmd + [src, dest]
result = self._run_command(cmd)
else:
bot.warning('%s does not exist, skipping.' %src)
def speak(self):
"""the client should announce self given that the shell is being used.
"""
if self._base is not None:
apps = " | ".join(self.apps())
bot.custom(prefix="%s %s" % (self, self._base),
message=apps,
color="CYAN")
else:
bot.info(self)
def main(args, parser, subparser):
from scif.main import ScifRecipe
apps = args.app
client = ScifRecipe(quiet=True, writable=False)
if len(apps) == 0:
bot.info("Usage: scif help <hello-world>")
for app in apps:
client.help(app)
def get_env(self, key=None):
'''return a particular value for an environment variable, if the key
exists. If no keys are defined, the entire environment is returned.
For an app-specific environment, use get_appenv.
'''
if key is not None:
if key in self.environment:
bot.info(self.environment[key])
return self.environment[key]
# otherwise, return the entire environment lookup
if hasattr(self, 'environment'):
return self.environment
def install_base(self):
''' create basic scif structure at the base for apps and metadata
Parameters
==========
base: the full path to the root folder to create /scif
'''
if not hasattr(self, '_base'):
bot.exit('Please set the base before installing to it.')
bot.info('Installing base at %s' % self._base)
mkdir_p(self.path_apps)
mkdir_p(self.path_data)
def install_base(self):
""" create basic scif structure at the base for apps and metadata
Parameters
==========
base: the full path to the root folder to create /scif
"""
if not hasattr(self, "_base"):
bot.exit("Please set the base before installing to it.")
bot.info("Installing base at %s" % self._base)
mkdir_p(self.path_apps)
mkdir_p(self.path_data)
def install_commands(self, app, settings, config):
'''install will finally, issue commands to install the app.
Parameters
==========
app should be the name of the app, for lookup in config['apps']
settings: the output of _init_app(), a dictionary of environment vars
config: should be the config for the app obtained with self.app(app)
'''
if "appinstall" in config:
cmd = '\n'.join(config['appinstall'])
bot.info('+ ' + 'appinstall '.ljust(5) + app)
os.system(cmd)
def preview_test(self, app, settings, config):
"""preview the test for the app
Parameters
==========
app should be the name of the app, for lookup in config['apps']
settings: the output of _init_app(), a dictionary of environment vars
config: should be the config for the app obtained with self.app(app)
"""
if "apptest" in config:
content = "\n".join(config["apptest"])
bot.info("+ " + "apptest ".ljust(5) + app)
print(settings["apptest"])
print(content)
def preview_environment(self, app, settings, config):
"""preview the environment section
Parameters
==========
app should be the name of the app, for lookup in config['apps']
settings: the output of _init_app(), a dictionary of environment vars
"""
content = ""
if "appenv" in config:
content = "\n".join(config["appenv"])
bot.info("+ " + "appenv ".ljust(5) + app)
print(settings["appenv"])
print(content)
return content
def preview_commands(self, app, settings, config):
"""install will finally, issue commands to install the app.
Parameters
==========
app should be the name of the app, for lookup in config['apps']
settings: the output of _init_app(), a dictionary of environment vars
config: should be the config for the app obtained with self.app(app)
"""
cmd = ""
if "appinstall" in config:
cmd = config["appinstall"]
bot.info("+ " + "appinstall ".ljust(5) + app)
print("\n".join(cmd))
return cmd
def preview_environment(self, app, settings, config):
'''preview the environment section
Parameters
==========
app should be the name of the app, for lookup in config['apps']
settings: the output of _init_app(), a dictionary of environment vars
'''
content = ''
if "appenv" in config:
content = '\n'.join(config['appenv'])
bot.info('+ ' + 'appenv '.ljust(5) + app)
print(settings['appenv'])
print(content)
return content
def preview_runscript(self, app, settings, config):
'''preview the runscript for an app
Parameters
==========
app should be the name of the app, for lookup in config['apps']
settings: the output of _init_app(), a dictionary of environment vars
config: should be the config for the app obtained with self.app(app)
'''
if "apprun" in config:
content = '\n'.join(config['apprun'])
bot.info('+ ' + 'apprun '.ljust(5) + app)
print(settings['apprun'])
print(settings['apphelp'])
print(content)
def install_script(self, section, app, settings, config):
'''a general function used by install_runscript, install_help, and
install_environment to write a script to a file from a config setting
section
Parameters
==========
section: should be the name of the section in the config (e.g., apprun)
app should be the name of the app, for lookup in config['apps']
settings: the output of _init_app(), a dictionary of environment vars
config: should be the config for the app obtained with self.app(app)
'''
if section in config:
content = '\n'.join(config[section])
bot.info('+ ' + section + ' '.ljust(5) + app)
write_file(settings[section], content)
def main(args, parser, subparser):
from scif.main import ScifRecipe
apps = args.recipe
if len(apps) > 0:
recipe = apps.pop(0)
if not os.path.exists(recipe):
bot.exit("Cannot find recipe file %s" % recipe)
client = ScifRecipe(recipe, writable=False)
# Preview the entire recipe, or the apps chosen
client.preview(apps)
else:
bot.info('You must provide a recipe file to preview!')
def run_command(self, cmd, spinner=True):
'''run_install will run a command (a list) and wrap in a spinner. A
result (dict) with message and return code is returned. If the
return value is not 0, an error is issed and we exit
'''
if spinner is True:
bot.spinner.start()
result = run_cmd(cmd)
if spinner is True:
bot.spinner.stop()
retval = result['return_code']
bot.info(result['message'])
if retval != 0:
bot.error('Return code %s' % retval)
sys.exit(retval)
return result
def exec(self, app=None):
'''exec is the underlying driver of both run and exec, taking a final
SCIF and executing the command for the user.
This function is called via self._exec, and the expectation is that
self.run or self.exec has been called first to prepare the environment
and SCIF settings.
If a user wants to preserve an environment variable from the console
it can be referenced with [e], for example $SCIF_DATA --> [e]SCIF_DATA
'''
name = ''
if app is not None and app in self.apps():
name = '[%s] ' % app
# If the entry folder still not set, we don't have an app
if self._entry_folder is None:
self._entry_folder = SCIF_BASE
# Change directory to the relevant place
os.chdir(self._entry_folder)
# export environment
runtime_environ = self.export_env()
# Execv to the entrypoint
executable = which(self._entry_point[0])
args = ''
if len(self._entry_point) > 1:
args = ' '.join(self._entry_point[1:])
cmd = "%s %s" % (executable, args)
bot.info('%sexecuting %s' % (name, cmd))
# Return output to console
process = os.popen(cmd)
while 1:
line = process.readline().strip('\n')
if not line: break
print(line)
def preview_labels(self, app, settings, config):
"""preview labels for an app
Parameters
==========
app should be the name of the app, for lookup in config['apps']
settings: the output of _init_app(), a dictionary of environment vars
config: should be the config for the app obtained with self.app(app)
"""
lookup = ""
if "applabels" in config:
labels = config["applabels"]
bot.info("+ " + "applabels ".ljust(5) + app)
print(settings["applabels"])
for line in labels:
label, value = get_parts(line, default="")
lookup += "%s=%s\n" % (label, value)
print(lookup)
return lookup
def install_labels(self, app, settings, config):
'''install labels will add labels to the app labelfile
Parameters
==========
app should be the name of the app, for lookup in config['apps']
settings: the output of _init_app(), a dictionary of environment vars
config: should be the config for the app obtained with self.app(app)
'''
lookup = dict()
if "applabels" in config:
labels = config['applabels']
bot.level
bot.info('+ ' + 'applabels '.ljust(5) + app)
for line in labels:
label, value = get_parts(line, default='')
lookup[label] = value
write_json(lookup, settings['applabels'])
return lookup
def preview_labels(self, app, settings, config):
'''preview labels for an app
Parameters
==========
app should be the name of the app, for lookup in config['apps']
settings: the output of _init_app(), a dictionary of environment vars
config: should be the config for the app obtained with self.app(app)
'''
lookup = ''
if "applabels" in config:
labels = config['appfiles']
bot.info('+ ' + 'applabels '.ljust(5) + app)
print(settings['applabels'])
for line in labels:
label, value = get_parts(line, default='')
lookup += '%s=%s\n' %(label,value)
print(lookup)
return lookup
def preview_files(self, app, settings, config):
"""install files will add files (or directories) to a destination.
If none specified, they are placed in the app base
Parameters
==========
app should be the name of the app, for lookup in config['apps']
settings: the output of _init_app(), a dictionary of environment vars
config: should be the config for the app obtained with self.app(app)
"""
if "appfiles" in config:
files = config["appfiles"]
bot.info("+ " + "appfiles ".ljust(5) + app)
for pair in files:
# Step 1: determine source and destination
src, dest = get_parts(pair, default=settings["approot"])
print("%s --> %s \n" % (src, dest))
def __init__(self, path=None, app=None, writable=True, quiet=False):
"""initialize the scientific filesystem by creating a scif folder
at the base, and loading the recipe to fill it.
Parameters
==========
path: is the first paramter, not required to initialize an empty
client session. The logic proceeds as follows:
1. If path is not defined, we want (empty) interactive session
2. We derive the base from the environment SCIF_BASE
"""
# 0. base determined from environment
from scif.defaults import SCIF_BASE
self.set_defaults()
self.quiet = quiet
# If recipe path not provided, try default base
if path is None:
path = SCIF_BASE
# 1. Determine if path is a recipe or base
if path is not None:
self.set_base(SCIF_BASE, writable=writable) # /scif
self.load(path, app, quiet) # recipe, environment
# 2. Neither, development client
else:
bot.info("[skeleton] session!")
bot.info(' load a recipe: client.load("recipe.scif")')
bot.info(' change default base: client.set_base("/")')
def install_script(self, section, app, settings, config, executable=False):
'''a general function used by install_runscript, install_help, and
install_environment to write a script to a file from a config setting
section
Parameters
==========
section: should be the name of the section in the config (e.g., apprun)
app should be the name of the app, for lookup in config['apps']
settings: the output of _init_app(), a dictionary of environment vars
config: should be the config for the app obtained with self.app(app)
executable: if the file is written, make it executable (defaults False)
'''
if section in config:
content = '\n'.join(config[section])
bot.info('+ ' + section + ' '.ljust(5) + app)
write_file(settings[section], content)
# Should we make the script executable (checks for exists)
if executable is True:
make_executable(settings[section])
def help(self, app):
"""print the help file for an app, if it exists.
Parameters
==========
app: the app to export variables for
"""
lines = None
if app in self.apps():
config = self.get_appenv(app)
if "SCIF_APPHELP" in config:
helpfile = config["SCIF_APPHELP"]
if os.path.exists(helpfile):
with open(helpfile, "r") as filey:
lines = filey.readlines()
print("".join(lines))
if lines is None:
bot.info("No help is defined for %s" % app)
return lines
def preview_recipe(self, app, settings, config):
"""Write the initial recipe for the app to its metadata folder.
Parameters
==========
app should be the name of the app, for lookup in config['apps']
settings: the output of _init_app(), a dictionary of environment vars
config: should be the config for the app obtained with self.app(app)
"""
recipe_file = settings["apprecipe"]
recipe = "# [scif] scientific filesystem recipe\n"
recipe += "# [date] %s\n" % datetime.now().strftime("%b-%d-%Y")
bot.info("+ " + "apprecipe ".ljust(5) + app)
print(recipe_file)
for section_name, section_content in config.items():
content = "%s\n" % "\n".join(section_content)
header = "%" + section_name + " %s" % app
recipe += "%s\n%s\n" % (header, content)
return recipe
def help(self, app):
'''print the help file for an app, if it exists.
Parameters
==========
app: the app to export variables for
'''
lines = None
if app in self.apps():
config = self.get_appenv(app)
if 'SCIF_APPHELP' in config:
helpfile = config['SCIF_APPHELP']
if os.path.exists(helpfile):
with open(helpfile, 'r') as filey:
lines = filey.readlines()
print(''.join(lines))
if lines is None:
bot.info('No help is defined for %s' % app)
return lines
def preview_recipe(self, app, settings, config):
'''Write the initial recipe for the app to its metadata folder.
Parameters
==========
app should be the name of the app, for lookup in config['apps']
settings: the output of _init_app(), a dictionary of environment vars
config: should be the config for the app obtained with self.app(app)
'''
recipe_file = settings['apprecipe']
recipe = '# [scif] scientific filesystem recipe\n'
recipe += '# [date] %s\n' % datetime.now().strftime('%b-%d-%Y')
bot.info('+ ' + 'apprecipe '.ljust(5) + app)
print(recipe_file)
for section_name, section_content in config.items():
content = "%s\n" % '\n'.join(section_content)
header = '%' + section_name + ' %s' % app
recipe += '%s\n%s\n' % (header, content)
return recipe