def validate_header(header, required_fields=None):
"""validate_header ensures that the first row contains the exp_id,
var_name, var_value, and token. Capitalization isn't important, but
ordering is. This criteria is very strict, but it's reasonable
to require.
Parameters
==========
header: the header row, as a list
required_fields: a list of required fields. We derive the required
length from this list.
Does not return, instead exits if malformed. Runs silently if OK.
"""
if required_fields is None:
required_fields = ["exp_id", "var_name", "var_value", "token"]
# The required length of the header based on required fields
length = len(required_fields)
# This is very strict, but no reason not to be
header = _validate_row(header, required_length=length)
header = [x.lower() for x in header]
for idx in range(length):
field = header[idx].lower().strip()
if required_fields[idx] != field:
bot.error("Malformed header field %s, exiting." % field)
sys.exit(1)
def validate_header(header, required_fields=None):
'''validate_header ensures that the first row contains the exp_id,
var_name, var_value, and token. Capitalization isn't important, but
ordering is. This criteria is very strict, but it's reasonable
to require.
Parameters
==========
header: the header row, as a list
required_fields: a list of required fields. We derive the required
length from this list.
Does not return, instead exits if malformed. Runs silently if OK.
'''
if required_fields is None:
required_fields = ['exp_id', 'var_name', 'var_value', 'token']
# The required length of the header based on required fields
length = len(required_fields)
# This is very strict, but no reason not to be
header = _validate_row(header, required_length=length)
header = [x.lower() for x in header]
for idx in range(length):
field = header[idx].lower().strip()
if required_fields[idx] != field:
bot.error('Malformed header field %s, exiting.' %field)
sys.exit(1)
def clone(url, tmpdir=None):
"""clone a repository from Github"""
if tmpdir is None:
tmpdir = tempfile.mkdtemp()
name = os.path.basename(url).replace(".git", "")
dest = "%s/%s" % (tmpdir, name)
return_code = os.system("git clone %s %s" % (url, dest))
if return_code == 0:
return dest
bot.error("Error cloning repo.")
sys.exit(return_code)
def clone(url, tmpdir=None):
'''clone a repository from Github'''
if tmpdir is None:
tmpdir = tempfile.mkdtemp()
name = os.path.basename(url).replace('.git', '')
dest = '%s/%s' % (tmpdir, name)
return_code = os.system('git clone %s %s' % (url, dest))
if return_code == 0:
return dest
bot.error('Error cloning repo.')
sys.exit(return_code)
def get_template(name, base=None):
'''read in and return a template file
'''
if base is None:
base = get_templatedir()
template_file = "%s/%s" % (base, name)
if os.path.exists(template_file):
with open(template_file, "r") as filey:
template = "".join(filey.readlines())
return template
bot.error("%s does not exist." % template_file)
def mkdir_p(path):
"""mkdir_p attempts to get the same functionality as mkdir -p
:param path: the path to create.
"""
try:
os.makedirs(path)
except OSError as e:
if e.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
bot.error("Error creating path %s, exiting." % path)
sys.exit(1)
def get_page(self, url, name='Chrome'):
'''get_page
open a particular url, checking for Timeout
'''
if self.browser is None:
self.browser = self.get_browser(name)
try:
return self.browser.get(url)
except TimeoutException:
bot.error('Browser request timeout. Are you connected to the internet?')
self.browser.close()
sys.exit(1)
def load_experiment(folder, return_path=False):
"""load_experiment:
reads in the config.json for a folder, returns None if not found.
:param folder: full path to experiment folder
:param return_path: if True, don't load the config.json, but return it
"""
fullpath = os.path.abspath(folder)
config = "%s/config.json" % (fullpath)
if not os.path.exists(config):
bot.error("config.json could not be found in %s" % (folder))
config = None
if return_path is False and config is not None:
config = read_json(config)
return config
def validate(self, url):
""" takes in a Github repository for validation of preview and
runtime (and possibly tests passing?
"""
# Preview must provide the live URL of the repository
if not url.startswith("http") or not "github" in url:
bot.error("Test of preview must be given a Github repostitory.")
return False
if not self._validate_preview(url):
return False
return True
def _validate_markdown(self, expfile):
"""ensure that fields are present in markdown file"""
try:
import yaml
except:
bot.error(
"Python yaml is required for testing yml/markdown files.")
sys.exit(1)
self.metadata = {}
uid = os.path.basename(expfile).strip(".md")
if os.path.exists(expfile):
with open(expfile, "r") as stream:
docs = yaml.load_all(stream)
for doc in docs:
if isinstance(doc, dict):
for k, v in doc.items():
print("%s: %s" % (k, v))
self.metadata[k] = v
self.metadata["uid"] = uid
fields = [
"github",
"preview",
"name",
"layout",
"tags",
"uid",
"maintainer",
]
# Tests for all fields
for field in fields:
if field not in self.metadata:
return False
if self.metadata[field] in ["", None]:
return False
if "github" not in self.metadata["github"]:
return notvalid("%s: not a valid github repository" % name)
if not isinstance(self.metadata["tags"], list):
return notvalid("%s: tags must be a list" % name)
if not re.search("(\w+://)(.+@)*([\w\d\.]+)(:[\d]+){0,1}/*(.*)",
self.metadata["github"]):
return notvalid("%s is not a valid URL." %
(self.metadata["github"]))
return True
def validate(self, url):
''' takes in a Github repository for validation of preview and
runtime (and possibly tests passing?
'''
# Preview must provide the live URL of the repository
if not url.startswith('http') or not 'github' in url:
bot.error('Test of preview must be given a Github repostitory.')
return False
if not self._validate_preview(url):
return False
return True
def load_experiment(folder, return_path=False):
'''load_experiment:
reads in the config.json for a folder, returns None if not found.
:param folder: full path to experiment folder
:param return_path: if True, don't load the config.json, but return it
'''
fullpath = os.path.abspath(folder)
config = "%s/config.json" %(fullpath)
if not os.path.exists(config):
bot.error("config.json could not be found in %s" %(folder))
config = None
if return_path is False and config is not None:
config = read_json(config)
return config
def validate(self, url):
''' takes in a Github repository for validation of preview and
runtime (and possibly tests passing?
'''
# Preview must provide the live URL of the repository
if not url.startswith('http') or not 'github' in url:
bot.error('Test of preview must be given a Github repostitory.')
return False
if not self._validate_preview(url):
return False
return True
def copy_directory(src, dest, force=False):
''' Copy an entire directory recursively
'''
if os.path.exists(dest) and force is True:
shutil.rmtree(dest)
try:
shutil.copytree(src, dest)
except OSError as e:
# If the error was caused because the source wasn't a directory
if e.errno == errno.ENOTDIR:
shutil.copy(src, dest)
else:
bot.error('Directory not copied. Error: %s' % e)
sys.exit(1)
def get_template(name, base=None):
"""read in and return a template file
"""
# If the file doesn't exist, assume relative to base
template_file = name
if not os.path.exists(template_file):
if base is None:
base = get_templatedir()
template_file = "%s/%s" % (base, name)
# Then try again, if it still doesn't exist, bad name
if os.path.exists(template_file):
with open(template_file, "r") as filey:
template = "".join(filey.readlines())
return template
bot.error("%s does not exist." % template_file)
def main(args,parser,subparser=None):
'''this is the main entrypoint for a container based web server, with
most of the variables coming from the environment. See the Dockerfile
template for how this function is executed.
'''
# First priority to args.base
base = args.base
if base is None:
base = os.environ.get('EXPFACTORY_BASE')
# Does the base folder exist?
if base is None:
bot.error("You must set a base of experiments with --base" % base)
sys.exit(1)
if not os.path.exists(base):
bot.error("Base folder %s does not exist." % base)
sys.exit(1)
# Export environment variables for the client
experiments = args.experiments
if experiments is None:
experiments = " ".join(glob("%s/*" % base))
os.environ['EXPFACTORY_EXPERIMENTS'] = experiments
# If defined and file exists, set runtime variables
if args.vars is not None:
if os.path.exists(args.vars):
os.environ['EXPFACTORY_RUNTIME_VARS'] = args.vars
os.environ['EXPFACTORY_RUNTIME_DELIM'] = args.delim
else:
bot.warning('Variables file %s not found.' %args.vars)
subid = os.environ.get('EXPFACTORY_STUDY_ID')
if args.subid is not None:
subid = args.subid
os.environ['EXPFACTORY_SUBID'] = subid
os.environ['EXPFACTORY_RANDOM'] = str(args.disable_randomize)
os.environ['EXPFACTORY_BASE'] = base
from expfactory.server import start
start(port=5000)
def main(args, parser, subparser=None):
"""this is the main entrypoint for a container based web server, with
most of the variables coming from the environment. See the Dockerfile
template for how this function is executed.
"""
# First priority to args.base
base = args.base
if base is None:
base = os.environ.get("EXPFACTORY_BASE")
# Does the base folder exist?
if base is None:
bot.error("You must set a base of experiments with --base" % base)
sys.exit(1)
if not os.path.exists(base):
bot.error("Base folder %s does not exist." % base)
sys.exit(1)
# Export environment variables for the client
experiments = args.experiments
if experiments is None:
experiments = " ".join(glob("%s/*" % base))
os.environ["EXPFACTORY_EXPERIMENTS"] = experiments
# If defined and file exists, set runtime variables
if args.vars is not None:
if os.path.exists(args.vars):
os.environ["EXPFACTORY_RUNTIME_VARS"] = args.vars
os.environ["EXPFACTORY_RUNTIME_DELIM"] = args.delim
else:
bot.warning("Variables file %s not found." % args.vars)
subid = os.environ.get("EXPFACTORY_STUDY_ID")
if args.subid is not None:
subid = args.subid
os.environ["EXPFACTORY_SUBID"] = subid
os.environ["EXPFACTORY_RANDOM"] = str(args.disable_randomize)
os.environ["EXPFACTORY_BASE"] = base
from expfactory.server import start
start(port=5000)
def _validate_markdown(self, expfile):
'''ensure that fields are present in markdown file'''
try:
import yaml
except:
bot.error(
'Python yaml is required for testing yml/markdown files.')
sys.exit(1)
self.metadata = {}
uid = os.path.basename(expfile).strip('.md')
if os.path.exists(expfile):
with open(expfile, "r") as stream:
docs = yaml.load_all(stream)
for doc in docs:
if isinstance(doc, dict):
for k, v in doc.items():
print('%s: %s' % (k, v))
self.metadata[k] = v
self.metadata['uid'] = uid
fields = [
'github', 'preview', 'name', 'layout', 'tags', 'uid',
'maintainer'
]
# Tests for all fields
for field in fields:
if field not in self.metadata:
return False
if self.metadata[field] in ['', None]:
return False
if 'github' not in self.metadata['github']:
return notvalid('%s: not a valid github repository' % name)
if not isinstance(self.metadata['tags'], list):
return notvalid('%s: tags must be a list' % name)
if not re.search("(\w+://)(.+@)*([\w\d\.]+)(:[\d]+){0,1}/*(.*)",
self.metadata['github']):
return notvalid('%s is not a valid URL.' %
(self.metadata['github']))
return True
def main():
parser = get_parser()
try:
args = parser.parse_args()
except:
sys.exit(0)
folder = args.folder
if folder is None:
folder = os.getcwd()
folder = os.path.abspath(folder)
survey = "%s/survey.tsv" % folder
if not os.path.exists(survey):
bot.error("Cannot find %s, required to generate survey." % survey)
sys.exit(1)
config = load_experiment(folder)
html, validation = generate_survey(config=config,
survey_file=survey,
form_action=args.action)
output = args.output
if output is None:
output = folder
output_index = "%s/index.html" % folder
if os.path.exists(output_index) and args.force is False:
bot.error("%s already exists, use --force to overwrite." %
output_index)
sys.exit(1)
bot.info("Writing output files to %s" % output_index)
template = get_template('survey/index.html')
template = sub_template(template, "{{html}}", html)
template = sub_template(template, "{{exp_id}}", config['exp_id'])
template = sub_template(template, "{{validation}}", validation)
write_file(output_index, template)
def getenv(variable_key, default=None, required=False, silent=True):
"""getenv will attempt to get an environment variable. If the variable
is not found, None is returned.
:param variable_key: the variable name
:param required: exit with error if not found
:param silent: Do not print debugging information for variable
"""
variable = os.environ.get(variable_key, default)
if variable is None and required:
bot.error("Cannot find environment variable %s, exiting." % variable_key)
sys.exit(1)
if not silent:
if variable is not None:
bot.verbose2("%s found as %s" % (variable_key, variable))
else:
bot.verbose2("%s not defined (None)" % variable_key)
return variable
def _validate_preview(self, url):
bot.test("Experiment url: %s" % url)
org, repo = url.split("/")[-2:]
if repo.endswith(".git"):
repo = repo.replace(".git", "")
github_pages = "https://%s.github.io/%s" % (org, repo)
bot.test("Github Pages url: %s" % github_pages)
response = requests.get(github_pages)
if response.status_code == 404:
bot.error("""Preview not found at %s. You must publish a static
preview from the master branch of your repository to
add to the library.""" % github_pages)
return False
index = response.text
tmpdir = tempfile.mkdtemp()
repo_master = clone(url, tmpdir)
contenders = glob("%s/*" % repo_master)
license = False
found = False
for test in contenders:
if os.path.isdir(test):
continue
print("...%s" % test)
if "LICENSE" in os.path.basename(test):
license = True
if os.path.basename(test) == "index.html":
bot.test("Found index file in repository.")
found = True
break
if license is False:
bot.warning(
"LICENSE file not found. This will be required for future experiments!"
)
self._print_valid(found)
return found
def _validate_preview(self, url):
bot.test('Experiment url: %s' %url)
org,repo = url.split('/')[-2:]
if repo.endswith('.git'):
repo = repo.replace('.git','')
github_pages = "https://%s.github.io/%s" %(org,repo)
bot.test('Github Pages url: %s' %github_pages)
response = requests.get(github_pages)
if response.status_code == 404:
bot.error('''Preview not found at %s. You must publish a static
preview from the master branch of your repository to
add to the library.''' % github_pages)
return False
index = response.text
tmpdir = tempfile.mkdtemp()
repo_master = clone(url, tmpdir)
contenders = glob('%s/*' %repo_master)
license = False
found = False
for test in contenders:
if os.path.isdir(test):
continue
print('...%s' %test)
if "LICENSE" in os.path.basename(test):
license = True
if os.path.basename(test) == "index.html":
bot.test('Found index file in repository.')
found = True
break
if license is False:
bot.warning("LICENSE file not found. This will be required for future experiments!")
self._print_valid(found)
return found
def _validate_markdown(self, expfile):
'''ensure that fields are present in markdown file'''
try:
import yaml
except:
bot.error('Python yaml is required for testing yml/markdown files.')
sys.exit(1)
self.metadata = {}
uid = os.path.basename(expfile).strip('.md')
if os.path.exists(expfile):
with open(expfile, "r") as stream:
docs = yaml.load_all(stream)
for doc in docs:
if isinstance(doc,dict):
for k,v in doc.items():
print('%s: %s' %(k,v))
self.metadata[k] = v
self.metadata['uid'] = uid
fields = ['github', 'preview', 'name', 'layout',
'tags', 'uid', 'maintainer']
# Tests for all fields
for field in fields:
if field not in self.metadata:
return False
if self.metadata[field] in ['',None]:
return False
if 'github' not in self.metadata['github']:
return notvalid('%s: not a valid github repository' % name)
if not isinstance(self.metadata['tags'],list):
return notvalid('%s: tags must be a list' % name)
if not re.search("(\w+://)(.+@)*([\w\d\.]+)(:[\d]+){0,1}/*(.*)", self.metadata['github']):
return notvalid('%s is not a valid URL.' %(self.metadata['github']))
return True
def main():
parser = get_parser()
try:
args = parser.parse_args()
except:
sys.exit(0)
print('Recruiting %s robot!' % args.robot)
folders = args.folders
if len(folders) == 0:
folders = [os.getcwd()]
# The drivers must be on path
here = os.path.abspath(os.path.dirname(__file__))
os.environ['PATH'] = "%s/drivers:%s" % (here, os.environ['PATH'])
# Load the robot!
if args.robot == 'jspsych':
from drivers.jspsych import JsPsychRobot as Robot
elif args.robot == 'survey':
from drivers.survey import SurveyRobot as Robot
robot = Robot(browser=args.browser, port=args.port, headless=args.headless)
for folder in folders:
folder = os.path.abspath(folder)
if not os.path.exists(folder):
bot.error("Cannot find %s, check that path exists." % folder)
else:
print('[folder] %s' % folder)
robot.validate(folder)
# Clean up shop!
robot.stop()
def main(args, parser, subparser=None):
'''this is the main entrypoint for a container based web server, with
most of the variables coming from the environment. See the Dockerfile
template for how this function is executed.
'''
# First priority to args.base
base = args.base
if base is None:
base = os.environ.get('EXPFACTORY_BASE')
# Does the base folder exist?
if base is None:
bot.error("You must set a base of experiments with --base" % base)
sys.exit(1)
if not os.path.exists(base):
bot.error("Base folder %s does not exist." % base)
sys.exit(1)
# Export environment variables for the client
experiments = args.experiments
if experiments is None:
experiments = " ".join(glob("%s/*" % base))
os.environ['EXPFACTORY_EXPERIMENTS'] = experiments
subid = os.environ.get('EXPFACTORY_STUDY_ID')
if args.subid is not None:
subid = args.subid
os.environ['EXPFACTORY_SUBID'] = subid
os.environ['EXPFACTORY_RANDOM'] = str(args.disable_randomize)
os.environ['EXPFACTORY_BASE'] = base
from expfactory.server import start
start(port=5000)
def main(args,parser,subparser):
folder = args.folder
if folder is None:
folder = os.getcwd()
source = args.src[0]
if source is None:
bot.error('Please provide a Github https address to install.')
sys.exit(1)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(source, cleanup=False)
exp_id = os.path.basename(source).replace('.git','')
if valid is True:
# Local Install
if os.path.exists(source):
config = load_experiment(source)
source = os.path.abspath(source)
else:
config = load_experiment("%s/%s" %(cli.tmpdir,exp_id))
source = "%s/%s" %(cli.tmpdir,exp_id)
exp_id = config['exp_id']
python_module = exp_id.replace('-','_').lower()
else:
bot.error('%s is not valid.' % exp_id)
sys.exit(1)
# Move static files to output folder
dest = "%s/%s" %(folder,exp_id)
bot.log("Installing %s to %s" %(exp_id, dest))
# Building container
in_container = False
if os.environ.get('SINGULARITY_IMAGE') is not None:
in_container = True
# Running, live container
elif os.environ.get('EXPFACTORY_CONTAINER') is not None:
in_container = True
if in_container is True:
# if in container, we always force
args.force = True
bot.log("Preparing experiment routes...")
template = get_template('experiments/template.py')
template = sub_template(template, '{{ exp_id }}', exp_id)
template = sub_template(template, '{{ exp_id_python }}', python_module)
# 1. Python blueprint
views = get_viewsdir(base=args.base)
view_output = "%s/%s.py" %(views, python_module)
save_template(view_output, template, base=views)
# 2. append to __init__
init = "%s/__init__.py" % views
with open(init,'a') as filey:
filey.writelines('from .%s import *\n' %python_module)
# 3. Instructions
if "instructions" in config:
instruct = "%s/%s.help" %(views, python_module)
with open(instruct,'w') as filey:
filey.writelines(config['instructions'])
if not os.path.exists(dest):
os.system('mkdir -p %s' %dest)
else:
if args.force is False:
bot.error('%s is not empty! Use --force to delete and re-create.' %folder)
sys.exit(1)
# We don't need to copy if experiment already there
if source != dest:
os.system('cp -R %s/* %s' %(source, dest))
def main(args, parser, subparser):
# List of experiments is required
template = get_template('build/docker/Dockerfile.template')
# For now, only one database provided
database = args.database
studyid = args.studyid
experiments = args.experiments
template = sub_template(template, "{{studyid}}", studyid)
template = sub_template(template, "{{database}}", database)
library = get_library(key='name')
apps = "\n"
# Add local experiments to library, first preference
local_installs = 0
for experiment in experiments:
if os.path.exists(experiment):
bot.info('local experiment %s found, validating...' % experiment)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(experiment, cleanup=False)
if valid is True:
local_installs += 1
config = load_experiment(experiment)
exp_id = config['exp_id']
# If we aren't building in the experiment directory, we need to copy there
output_dir = "%s/%s" % (os.path.abspath(
os.path.dirname(args.output)), exp_id)
experiment_dir = os.path.abspath(experiment)
if output_dir != experiment_dir:
copy_directory(experiment_dir, output_dir)
config['local'] = os.path.abspath(experiment)
library[exp_id] = config
# Warn the user that local installs are not reproducible (from recipe)
if local_installs > 0:
bot.warning(
"%s local installs detected: build is not reproducible without experiment folders"
% local_installs)
# Build Image with Experiments
for experiment in experiments:
exp_id = os.path.basename(experiment)
if exp_id in library:
config = library[exp_id]
app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id)
# Here add custom build routine, should be list of lines
if "install" in config:
commands = "\n".join(
["RUN %s " % s for x in config['install']]).strip('\n')
app = "%s%s\n" % (app, commands)
# The final installation step, either from Github (url) or local folder
if "local" in config:
app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id)
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app, exp_id)
else:
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app, config['github'])
apps = "%s%s\n" % (apps, app)
else:
bot.warning('%s not in library, check spelling & punctuation.' %
exp_id)
if apps == "\n":
bot.error('No valid experiments found, cancelling recipe build.')
sys.exit(1)
template = sub_template(template, "{{experiments}}", apps)
outfile = write_file(args.output, template)
bot.log("Recipe written to %s" % outfile)
def main(args,parser,subparser):
template = 'build/docker/Dockerfile.template'
# Full path to template is required if provided via input
if args.input is not None:
template = args.input
template = get_template(template)
# For now, only one database provided
database = args.database
studyid = args.studyid
experiments = args.experiments
branch = "-b %s" %os.environ.get('EXPFACTORY_BRANCH','master')
headless = "false"
if args.headless is True:
headless = "true"
template = sub_template(template,"{{studyid}}",studyid)
template = sub_template(template,"{{database}}",database)
template = sub_template(template,"{{headless}}",headless)
template = sub_template(template,"{{branch}}",branch)
if args.headless is True:
bot.info("Headless build detected, you will need to generate tokens for application entry with expfactory users --new")
library = get_library(key='name')
apps = "\n"
# Add local experiments to library, first preference
local_installs = 0
for experiment in experiments:
if os.path.exists(experiment):
bot.info('local experiment %s found, validating...' %experiment)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(experiment, cleanup=False)
if valid is True:
local_installs +=1
config = load_experiment(experiment)
exp_id = config['exp_id']
# If we aren't building in the experiment directory, we need to copy there
output_dir = "%s/%s" %(os.path.abspath(os.path.dirname(args.output)), exp_id)
experiment_dir = os.path.abspath(experiment)
if output_dir != experiment_dir:
copy_directory(experiment_dir, output_dir)
config['local'] = os.path.abspath(experiment)
library[exp_id] = config
# Warn the user that local installs are not reproducible (from recipe)
if local_installs > 0:
bot.warning("%s local installs detected: build is not reproducible without experiment folders" %local_installs)
# Build Image with Experiments
for experiment in experiments:
exp_id = os.path.basename(experiment)
if exp_id in library:
config = library[exp_id]
app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" %(exp_id, exp_id)
# Here add custom build routine, should be list of lines
if "install" in config:
commands = "\n".join(["RUN %s "%s for x in config['install']]).strip('\n')
app = "%s%s\n" %(app, commands)
# The final installation step, either from Github (url) or local folder
if "local" in config:
app = "%sADD %s /scif/apps/%s\n" %(app, exp_id, exp_id)
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, exp_id)
else:
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, config['github'])
apps = "%s%s\n" %(apps,app)
else:
bot.warning('%s not in library, check spelling & punctuation.' %exp_id)
if apps == "\n":
bot.error('No valid experiments found, cancelling recipe build.')
sys.exit(1)
template = sub_template(template,"{{experiments}}",apps)
outfile = write_file(args.output,template)
bot.log("Recipe written to %s" %outfile)
def main(args, parser, subparser):
folder = args.folder
if folder is None:
folder = os.getcwd()
source = args.src[0]
if source is None:
bot.error('Please provide a Github https address to install.')
sys.exit(1)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(source, cleanup=False)
exp_id = os.path.basename(source).replace('.git', '')
if valid is True:
# Local Install
if os.path.exists(source):
config = load_experiment(source)
source = os.path.abspath(source)
else:
config = load_experiment("%s/%s" % (cli.tmpdir, exp_id))
source = "%s/%s" % (cli.tmpdir, exp_id)
exp_id = config['exp_id']
python_module = exp_id.replace('-', '_').lower()
else:
bot.error('%s is not valid.' % exp_id)
sys.exit(1)
# Move static files to output folder
dest = "%s/%s" % (folder, exp_id)
bot.log("Installing %s to %s" % (exp_id, dest))
# Building container
in_container = False
if os.environ.get('SINGULARITY_IMAGE') is not None:
in_container = True
# Running, live container
elif os.environ.get('EXPFACTORY_CONTAINER') is not None:
in_container = True
if in_container is True:
# if in container, we always force
args.force = True
bot.log("Preparing experiment routes...")
template = get_template('experiments/template.py')
template = sub_template(template, '{{ exp_id }}', exp_id)
template = sub_template(template, '{{ exp_id_python }}', python_module)
# 1. Python blueprint
views = get_viewsdir(base=args.base)
view_output = "%s/%s.py" % (views, python_module)
save_template(view_output, template, base=views)
# 2. append to __init__
init = "%s/__init__.py" % views
with open(init, 'a') as filey:
filey.writelines('from .%s import *\n' % python_module)
# 3. Instructions
if "instructions" in config:
instruct = "%s/%s.help" % (views, python_module)
with open(instruct, 'w') as filey:
filey.writelines(config['instructions'])
if not os.path.exists(dest):
os.system('mkdir -p %s' % dest)
else:
if args.force is False:
bot.error('%s is not empty! Use --force to delete and re-create.' %
folder)
sys.exit(1)
# We don't need to copy if experiment already there
if source != dest:
os.system('cp -R %s/* %s' % (source, dest))
def main(args, parser, subparser):
template = "build/docker/Dockerfile.template"
# Full path to template is required if provided via input
if args.input is not None:
template = args.input
template = get_template(template)
# For now, only one database provided
database = args.database
studyid = args.studyid
experiments = args.experiments
branch = "-b %s" % os.environ.get("EXPFACTORY_BRANCH", "master")
headless = "false"
if args.headless is True:
headless = "true"
template = sub_template(template, "{{studyid}}", studyid)
template = sub_template(template, "{{database}}", database)
template = sub_template(template, "{{headless}}", headless)
template = sub_template(template, "{{branch}}", branch)
if args.headless is True:
bot.info(
"Headless build detected, you will need to generate tokens for application entry with expfactory users --new"
)
library = get_library(key="name")
apps = "\n"
# Add local experiments to library, first preference
local_installs = 0
for experiment in experiments:
if os.path.exists(experiment):
bot.info("local experiment %s found, validating..." % experiment)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(experiment, cleanup=False)
if valid is True:
local_installs += 1
config = load_experiment(experiment)
exp_id = config["exp_id"]
# If we aren't building in the experiment directory, we need to copy there
output_dir = "%s/%s" % (
os.path.abspath(os.path.dirname(args.output)),
exp_id,
)
experiment_dir = os.path.abspath(experiment)
if output_dir != experiment_dir:
copy_directory(experiment_dir, output_dir)
config["local"] = os.path.abspath(experiment)
library[exp_id] = config
# Warn the user that local installs are not reproducible (from recipe)
if local_installs > 0:
bot.warning(
"%s local installs detected: build is not reproducible without experiment folders"
% local_installs
)
# Build Image with Experiments
for experiment in experiments:
exp_id = os.path.basename(experiment)
if exp_id in library:
config = library[exp_id]
app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id)
# Here add custom build routine, should be list of lines
if "install" in config:
commands = "\n".join(["RUN %s " % s for x in config["install"]]).strip(
"\n"
)
app = "%s%s\n" % (app, commands)
# The final installation step, either from Github (url) or local folder
if "local" in config:
app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id)
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app,
exp_id,
)
else:
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app,
config["github"],
)
apps = "%s%s\n" % (apps, app)
else:
bot.warning("%s not in library, check spelling & punctuation." % exp_id)
if apps == "\n":
bot.error("No valid experiments found, cancelling recipe build.")
sys.exit(1)
template = sub_template(template, "{{experiments}}", apps)
outfile = write_file(args.output, template)
bot.log("Recipe written to %s" % outfile)
