def setup(self):
""" obtain database and filesystem preferences from defaults,
and compare with selection in container.
"""
self.selection = EXPFACTORY_EXPERIMENTS
self.ordered = len(EXPFACTORY_EXPERIMENTS) > 0
self.data_base = EXPFACTORY_DATA
self.study_id = EXPFACTORY_SUBID
self.base = EXPFACTORY_BASE
self.randomize = EXPFACTORY_RANDOMIZE
self.headless = EXPFACTORY_HEADLESS
# Generate variables, if they exist
self.vars = generate_runtime_vars() or None
available = get_experiments("%s" % self.base)
self.experiments = get_selection(available, self.selection)
self.logger.debug(self.experiments)
self.lookup = make_lookup(self.experiments)
final = "\n".join(list(self.lookup.keys()))
bot.log("Headless mode: %s" % self.headless)
bot.log("User has selected: %s" % self.selection)
bot.log("Experiments Available: %s" % "\n".join(available))
bot.log("Randomize: %s" % self.randomize)
bot.log("Final Set \n%s" % final)
def setup(self):
''' obtain database and filesystem preferences from defaults,
and compare with selection in container.
'''
self.selection = EXPFACTORY_EXPERIMENTS
self.ordered = len(EXPFACTORY_EXPERIMENTS) > 0
self.data_base = EXPFACTORY_DATA
self.study_id = EXPFACTORY_SUBID
self.base = EXPFACTORY_BASE
self.randomize = EXPFACTORY_RANDOMIZE
self.headless = EXPFACTORY_HEADLESS
# Generate variables, if they exist
self.vars = generate_runtime_vars() or None
available = get_experiments("%s" % self.base)
self.experiments = get_selection(available, self.selection)
self.logger.debug(self.experiments)
self.lookup = make_lookup(self.experiments)
final = "\n".join(list(self.lookup.keys()))
bot.log("Headless mode: %s" % self.headless)
bot.log("User has selected: %s" % self.selection)
bot.log("Experiments Available: %s" %"\n".join(available))
bot.log("Randomize: %s" % self.randomize)
bot.log("Final Set \n%s" % final)
def _forward_choices(self, block):
'''move through one or more choices in a timeline, including a writing
block
'''
choices = block["choices"]
if choices and len(choices) > 0:
try:
random_choice = choice(choices)
continue_key = self.key_lookup(random_choice)
self.browser.find_element_by_tag_name('html').send_keys(
continue_key)
except ValueError:
bot.log("ValueError, %s found as choices." % (choices))
else:
self.browser.find_element_by_tag_name('html').send_keys(Keys.ENTER)
def validate(self, folder):
'''validate is the first entrypoint function for running an experiment
or survey robot. It ensures that the content is valid,
and then calls _validate (should be defined in subclass)'''
validator = ExperimentValidator()
valid = validator.validate(folder)
if valid is True:
# IF missing favicon, add
self._check_favicon(folder)
valid = self._validate(folder)
bot.log("[done] stopping web server...")
self.httpd.server_close()
else:
bot.warning('%s is not valid, skipping robot testing.' %folder)
def initdb(self):
'''initdb will check for writability of the data folder, meaning
that it is bound to the local machine. If the folder isn't bound,
expfactory runs in demo mode (not saving data)
'''
self.database = EXPFACTORY_DATABASE
bot.info("DATABASE: %s" %self.database)
# Supported database options
valid = ('sqlite', 'postgres', 'mysql', 'filesystem')
if not self.database.startswith(valid):
bot.warning('%s is not yet a supported type, saving to filesystem.' % self.database)
self.database = 'filesystem'
# Add functions specific to database type
self.init_db() # uses url in self.database
bot.log("Data base: %s" % self.database)
def initdb(self):
'''initdb will check for writability of the data folder, meaning
that it is bound to the local machine. If the folder isn't bound,
expfactory runs in demo mode (not saving data)
'''
self.database = EXPFACTORY_DATABASE
bot.info("DATABASE: %s" %self.database)
# Supported database options
valid = ('sqlite', 'postgres', 'mysql', 'filesystem')
if not self.database.startswith(valid):
bot.warning('%s is not yet a supported type, saving to filesystem.' % self.database)
self.database = 'filesystem'
# Add functions specific to database type
self.init_db() # uses url in self.database
bot.log("Data base: %s" % self.database)
def _validate(self, folder):
''''validate is subclass specific function to validate a survey.
it is called by ExpfactoryRobot in primary function "validate"
'''
base = os.path.abspath(folder)
# Set up a web browser
os.chdir(base)
self.get_browser()
self.browser.implicitly_wait(
3) # if error, will wait 3 seconds and retry
self.browser.set_page_load_timeout(10)
bot.log("STARTING TEST OF EXPERIMENT")
self.get_page("http://localhost:%s/" % self.port)
sleep(3)
while True:
try:
wait_time, finished = self.test_block()
if finished is True:
break
except UnexpectedAlertPresentException:
bot.log("Found alert: closing.")
try:
alert = self.browser.switch_to_alert()
alert.accept()
except:
pass
bot.log("FINISHING TEST OF EXPERIMENT")
def _validate(self, folder):
''''validate is subclass specific function to validate a survey.
it is called by ExpfactoryRobot in primary function "validate"
'''
if self.started is False:
self.get_server()
base = os.path.abspath(folder)
# Set up a web browser
os.chdir(base)
self.get_browser()
self.browser.implicitly_wait(
3) # if error, will wait 3 seconds and retry
self.browser.set_page_load_timeout(10)
bot.log("STARTING TEST OF SURVEY")
self.get_page("http://localhost:%s" % self.port)
sleep(3)
count = 1
while True:
bot.log("Testing page %s" % count)
try:
finished = self.advance_survey()
if finished is True:
break
count += 1
except UnexpectedAlertPresentException:
bot.log("Found alert: closing.")
try:
alert = browser.switch_to_alert()
alert.accept()
except:
pass
bot.log("FINISHING TEST OF SURVEY")
# Stop the server
self.httpd.server_close()
def setup(self):
''' obtain database and filesystem preferences from defaults,
and compare with selection in container.
'''
self.selection = EXPFACTORY_EXPERIMENTS
self.data_base = EXPFACTORY_DATA
self.study_id = EXPFACTORY_SUBID
self.base = EXPFACTORY_BASE
self.randomize = EXPFACTORY_RANDOMIZE
available = get_experiments("%s" % self.base)
self.experiments = get_selection(available, self.selection)
self.logger.debug(self.experiments)
self.lookup = make_lookup(self.experiments)
final = "\n".join(list(self.lookup.keys()))
bot.log("User has selected: %s" % self.selection)
bot.log("Experiments Available: %s" % "\n".join(available))
bot.log("Randomize: %s" % self.randomize)
bot.log("Final Set \n%s" % final)
def main(args, parser, subparser):
folder = args.folder
if folder is None:
folder = os.getcwd()
source = args.src[0]
if source is None:
bot.error('Please provide a Github https address to install.')
sys.exit(1)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(source, cleanup=False)
exp_id = os.path.basename(source).replace('.git', '')
if valid is True:
# Local Install
if os.path.exists(source):
config = load_experiment(source)
source = os.path.abspath(source)
else:
config = load_experiment("%s/%s" % (cli.tmpdir, exp_id))
source = "%s/%s" % (cli.tmpdir, exp_id)
exp_id = config['exp_id']
python_module = exp_id.replace('-', '_').lower()
else:
bot.error('%s is not valid.' % exp_id)
sys.exit(1)
# Move static files to output folder
dest = "%s/%s" % (folder, exp_id)
bot.log("Installing %s to %s" % (exp_id, dest))
# Building container
in_container = False
if os.environ.get('SINGULARITY_IMAGE') is not None:
in_container = True
# Running, live container
elif os.environ.get('EXPFACTORY_CONTAINER') is not None:
in_container = True
if in_container is True:
# if in container, we always force
args.force = True
bot.log("Preparing experiment routes...")
template = get_template('experiments/template.py')
template = sub_template(template, '{{ exp_id }}', exp_id)
template = sub_template(template, '{{ exp_id_python }}', python_module)
# 1. Python blueprint
views = get_viewsdir(base=args.base)
view_output = "%s/%s.py" % (views, python_module)
save_template(view_output, template, base=views)
# 2. append to __init__
init = "%s/__init__.py" % views
with open(init, 'a') as filey:
filey.writelines('from .%s import *\n' % python_module)
# 3. Instructions
if "instructions" in config:
instruct = "%s/%s.help" % (views, python_module)
with open(instruct, 'w') as filey:
filey.writelines(config['instructions'])
if not os.path.exists(dest):
os.system('mkdir -p %s' % dest)
else:
if args.force is False:
bot.error('%s is not empty! Use --force to delete and re-create.' %
folder)
sys.exit(1)
# We don't need to copy if experiment already there
if source != dest:
os.system('cp -R %s/* %s' % (source, dest))
def main(args, parser, subparser):
template = "build/docker/Dockerfile.template"
# Full path to template is required if provided via input
if args.input is not None:
template = args.input
template = get_template(template)
# For now, only one database provided
database = args.database
studyid = args.studyid
experiments = args.experiments
branch = "-b %s" % os.environ.get("EXPFACTORY_BRANCH", "master")
headless = "false"
if args.headless is True:
headless = "true"
template = sub_template(template, "{{studyid}}", studyid)
template = sub_template(template, "{{database}}", database)
template = sub_template(template, "{{headless}}", headless)
template = sub_template(template, "{{branch}}", branch)
if args.headless is True:
bot.info(
"Headless build detected, you will need to generate tokens for application entry with expfactory users --new"
)
library = get_library(key="name")
apps = "\n"
# Add local experiments to library, first preference
local_installs = 0
for experiment in experiments:
if os.path.exists(experiment):
bot.info("local experiment %s found, validating..." % experiment)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(experiment, cleanup=False)
if valid is True:
local_installs += 1
config = load_experiment(experiment)
exp_id = config["exp_id"]
# If we aren't building in the experiment directory, we need to copy there
output_dir = "%s/%s" % (
os.path.abspath(os.path.dirname(args.output)),
exp_id,
)
experiment_dir = os.path.abspath(experiment)
if output_dir != experiment_dir:
copy_directory(experiment_dir, output_dir)
config["local"] = os.path.abspath(experiment)
library[exp_id] = config
# Warn the user that local installs are not reproducible (from recipe)
if local_installs > 0:
bot.warning(
"%s local installs detected: build is not reproducible without experiment folders"
% local_installs
)
# Build Image with Experiments
for experiment in experiments:
exp_id = os.path.basename(experiment)
if exp_id in library:
config = library[exp_id]
app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id)
# Here add custom build routine, should be list of lines
if "install" in config:
commands = "\n".join(["RUN %s " % s for x in config["install"]]).strip(
"\n"
)
app = "%s%s\n" % (app, commands)
# The final installation step, either from Github (url) or local folder
if "local" in config:
app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id)
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app,
exp_id,
)
else:
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app,
config["github"],
)
apps = "%s%s\n" % (apps, app)
else:
bot.warning("%s not in library, check spelling & punctuation." % exp_id)
if apps == "\n":
bot.error("No valid experiments found, cancelling recipe build.")
sys.exit(1)
template = sub_template(template, "{{experiments}}", apps)
outfile = write_file(args.output, template)
bot.log("Recipe written to %s" % outfile)
def main(args,parser,subparser):
template = 'build/docker/Dockerfile.template'
# Full path to template is required if provided via input
if args.input is not None:
template = args.input
template = get_template(template)
# For now, only one database provided
database = args.database
studyid = args.studyid
experiments = args.experiments
branch = "-b %s" %os.environ.get('EXPFACTORY_BRANCH','master')
headless = "false"
if args.headless is True:
headless = "true"
template = sub_template(template,"{{studyid}}",studyid)
template = sub_template(template,"{{database}}",database)
template = sub_template(template,"{{headless}}",headless)
template = sub_template(template,"{{branch}}",branch)
if args.headless is True:
bot.info("Headless build detected, you will need to generate tokens for application entry with expfactory users --new")
library = get_library(key='name')
apps = "\n"
# Add local experiments to library, first preference
local_installs = 0
for experiment in experiments:
if os.path.exists(experiment):
bot.info('local experiment %s found, validating...' %experiment)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(experiment, cleanup=False)
if valid is True:
local_installs +=1
config = load_experiment(experiment)
exp_id = config['exp_id']
# If we aren't building in the experiment directory, we need to copy there
output_dir = "%s/%s" %(os.path.abspath(os.path.dirname(args.output)), exp_id)
experiment_dir = os.path.abspath(experiment)
if output_dir != experiment_dir:
copy_directory(experiment_dir, output_dir)
config['local'] = os.path.abspath(experiment)
library[exp_id] = config
# Warn the user that local installs are not reproducible (from recipe)
if local_installs > 0:
bot.warning("%s local installs detected: build is not reproducible without experiment folders" %local_installs)
# Build Image with Experiments
for experiment in experiments:
exp_id = os.path.basename(experiment)
if exp_id in library:
config = library[exp_id]
app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" %(exp_id, exp_id)
# Here add custom build routine, should be list of lines
if "install" in config:
commands = "\n".join(["RUN %s "%s for x in config['install']]).strip('\n')
app = "%s%s\n" %(app, commands)
# The final installation step, either from Github (url) or local folder
if "local" in config:
app = "%sADD %s /scif/apps/%s\n" %(app, exp_id, exp_id)
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, exp_id)
else:
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, config['github'])
apps = "%s%s\n" %(apps,app)
else:
bot.warning('%s not in library, check spelling & punctuation.' %exp_id)
if apps == "\n":
bot.error('No valid experiments found, cancelling recipe build.')
sys.exit(1)
template = sub_template(template,"{{experiments}}",apps)
outfile = write_file(args.output,template)
bot.log("Recipe written to %s" %outfile)
def main(args, parser, subparser):
# List of experiments is required
template = get_template('build/docker/Dockerfile.template')
# For now, only one database provided
database = args.database
studyid = args.studyid
experiments = args.experiments
template = sub_template(template, "{{studyid}}", studyid)
template = sub_template(template, "{{database}}", database)
library = get_library(key='name')
apps = "\n"
# Add local experiments to library, first preference
local_installs = 0
for experiment in experiments:
if os.path.exists(experiment):
bot.info('local experiment %s found, validating...' % experiment)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(experiment, cleanup=False)
if valid is True:
local_installs += 1
config = load_experiment(experiment)
exp_id = config['exp_id']
# If we aren't building in the experiment directory, we need to copy there
output_dir = "%s/%s" % (os.path.abspath(
os.path.dirname(args.output)), exp_id)
experiment_dir = os.path.abspath(experiment)
if output_dir != experiment_dir:
copy_directory(experiment_dir, output_dir)
config['local'] = os.path.abspath(experiment)
library[exp_id] = config
# Warn the user that local installs are not reproducible (from recipe)
if local_installs > 0:
bot.warning(
"%s local installs detected: build is not reproducible without experiment folders"
% local_installs)
# Build Image with Experiments
for experiment in experiments:
exp_id = os.path.basename(experiment)
if exp_id in library:
config = library[exp_id]
app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id)
# Here add custom build routine, should be list of lines
if "install" in config:
commands = "\n".join(
["RUN %s " % s for x in config['install']]).strip('\n')
app = "%s%s\n" % (app, commands)
# The final installation step, either from Github (url) or local folder
if "local" in config:
app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id)
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app, exp_id)
else:
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app, config['github'])
apps = "%s%s\n" % (apps, app)
else:
bot.warning('%s not in library, check spelling & punctuation.' %
exp_id)
if apps == "\n":
bot.error('No valid experiments found, cancelling recipe build.')
sys.exit(1)
template = sub_template(template, "{{experiments}}", apps)
outfile = write_file(args.output, template)
bot.log("Recipe written to %s" % outfile)
def main(args,parser,subparser):
folder = args.folder
if folder is None:
folder = os.getcwd()
source = args.src[0]
if source is None:
bot.error('Please provide a Github https address to install.')
sys.exit(1)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(source, cleanup=False)
exp_id = os.path.basename(source).replace('.git','')
if valid is True:
# Local Install
if os.path.exists(source):
config = load_experiment(source)
source = os.path.abspath(source)
else:
config = load_experiment("%s/%s" %(cli.tmpdir,exp_id))
source = "%s/%s" %(cli.tmpdir,exp_id)
exp_id = config['exp_id']
python_module = exp_id.replace('-','_').lower()
else:
bot.error('%s is not valid.' % exp_id)
sys.exit(1)
# Move static files to output folder
dest = "%s/%s" %(folder,exp_id)
bot.log("Installing %s to %s" %(exp_id, dest))
# Building container
in_container = False
if os.environ.get('SINGULARITY_IMAGE') is not None:
in_container = True
# Running, live container
elif os.environ.get('EXPFACTORY_CONTAINER') is not None:
in_container = True
if in_container is True:
# if in container, we always force
args.force = True
bot.log("Preparing experiment routes...")
template = get_template('experiments/template.py')
template = sub_template(template, '{{ exp_id }}', exp_id)
template = sub_template(template, '{{ exp_id_python }}', python_module)
# 1. Python blueprint
views = get_viewsdir(base=args.base)
view_output = "%s/%s.py" %(views, python_module)
save_template(view_output, template, base=views)
# 2. append to __init__
init = "%s/__init__.py" % views
with open(init,'a') as filey:
filey.writelines('from .%s import *\n' %python_module)
# 3. Instructions
if "instructions" in config:
instruct = "%s/%s.help" %(views, python_module)
with open(instruct,'w') as filey:
filey.writelines(config['instructions'])
if not os.path.exists(dest):
os.system('mkdir -p %s' %dest)
else:
if args.force is False:
bot.error('%s is not empty! Use --force to delete and re-create.' %folder)
sys.exit(1)
# We don't need to copy if experiment already there
if source != dest:
os.system('cp -R %s/* %s' %(source, dest))