def get_concat_js(valid_experiments):
'''get_concat_js
Return javascript concat section for valid experiments, based on psiturk.json
:param valid_experiments: full paths to valid experiments to include
..note::
case "simple-rt":
experiments = experiments.concat(simple-rt_experiment)
break;
case "choice-rt":
experiments = experiments.concat(choice-rt_experiment)
break;
Format for experiment variables is [exp_id]_experiment
'''
concatjs = "\n"
for valid_experiment in valid_experiments:
experiment = load_experiment(valid_experiment)[0]
tag = str(experiment["exp_id"])
concatjs = '%scase "%s":\n' %(concatjs,tag)
concatjs = '%s experiments = experiments.concat(%s_experiment)\n' %(concatjs,tag)
concatjs = '%s break;\n' %(concatjs)
return concatjs
def get_concat_js(valid_experiments):
'''get_concat_js
Return javascript concat section for valid experiments, based on psiturk.json
:param valid_experiments: full paths to valid experiments to include
..note::
case "simple-rt":
experiments = experiments.concat(simple-rt_experiment)
break;
case "choice-rt":
experiments = experiments.concat(choice-rt_experiment)
break;
Format for experiment variables is [exp_id]_experiment
'''
concatjs = "\n"
for valid_experiment in valid_experiments:
experiment = load_experiment(valid_experiment)[0]
tag = str(experiment["exp_id"])
concatjs = '%scase "%s":\n' % (concatjs, tag)
concatjs = '%s experiments = experiments.concat(%s_experiment)\n' % (
concatjs, tag)
concatjs = '%s break;\n' % (concatjs)
return concatjs
def validate_surveys(survey_tags,
survey_repo,
survey_file="survey.tsv",
delim="\t"):
'''validate_surveys validates an experiment factory survey folder
:param survey_tags: a list of surveys to validate
:param survey_repo: the survey repo with the survey folders to validate
:param survey_file: the default file to validate
'''
if isinstance(survey_tags, str):
survey_tags = [survey_tags]
for survey_tag in survey_tags:
survey_folder = "%s/%s" % (survey_repo, survey_tag)
print "Testing load of config.json for %s" % (survey_folder)
survey = load_experiment("%s" % survey_folder)
survey_questions = "%s/%s" % (survey_folder, survey_file)
print "Testing valid columns in %s" % (survey[0]["exp_id"])
df = read_survey_file(survey_questions, delim=delim)
assert_equal(isinstance(df, pandas.DataFrame), True)
print "Testing survey generation of %s" % (survey[0]["exp_id"])
questions, required_count = parse_questions(survey_questions,
exp_id=survey[0]["exp_id"],
validate=True)
print "Testing validation generation of %s" % (survey[0]["exp_id"])
validation = parse_validation(required_count)
def validate_experiment_tag(experiment_folder):
'''validate_experiment_tag looks for definition of exp_id as the tag somewhere in experiment.js. We are only requiring one definition for now (a more lax approach), but this standard might be changed.
'''
experiments = find_directories(experiment_folder)
print "Testing %s experiment for definition of exp_id in experiment.js..."
for contender in experiments:
if validate(contender,warning=False) == True:
experiment = load_experiment(contender)
tag = experiment[0]["exp_id"]
# Experiment MUST contain experiment.js to run main experiment
print "TESTING %s for exp_id in experiment.js..." %tag
assert_equal("experiment.js" in experiment[0]["run"],True)
if "experiment.js" in experiment[0]["run"]:
experiment_js_file = open("%s/%s/experiment.js" %(experiment_folder,tag),"r")
experiment_js_list = [x.strip("\n").replace("'","").replace('"',"").replace(" ","") for x in experiment_js_file.readlines()]
experiment_js_file.close()
experiment_js = "".join(experiment_js_list)
[x]
has_exp_id = re.search("exp_id:%s" %tag,experiment_js) != None or re.search("exp_id=%s" %tag,experiment_js) != None
assert_equal(has_exp_id,True)
# Ensure all are formatted correctly
exp_id_instances = [re.findall("exp_id[=|:].+",x) for x in experiment_js_list if len(re.findall("exp_id[=|:].+,",x)) != 0]
line_numbers = [x+1 for x in range(len(experiment_js_list)) if len(re.findall("exp_id[=|:].+,",experiment_js_list[x])) != 0]
for e in range(len(exp_id_instances)):
exp_id_instance = exp_id_instances[e]
line_number = line_numbers[e]
print "Checking %s on line %s..." %(exp_id_instance[0],line_number)
assert_equal(re.search(tag,exp_id_instance[0])!=None,True)
def validate_surveys(survey_tags,survey_repo,survey_file="survey.tsv",delim="\t",raise_error=True):
'''validate_surveys validates an experiment factory survey folder
:param survey_tags: a list of surveys to validate
:param survey_repo: the survey repo with the survey folders to validate
:param survey_file: the default file to validate
:param raise_error: throw the error (meaning don't return True or False instead)
'''
if isinstance(survey_tags,str):
survey_tags = [survey_tags]
for survey_tag in survey_tags:
try:
survey_folder = "%s/%s" %(survey_repo,survey_tag)
print "Testing load of config.json for %s" %(survey_folder)
survey = load_experiment("%s" %survey_folder)
survey_questions = "%s/%s" %(survey_folder,survey_file)
print "Testing valid columns in %s" %(survey[0]["exp_id"])
df = read_survey_file(survey_questions,delim=delim)
assert_equal(isinstance(df,pandas.DataFrame),True)
print "Testing survey generation of %s" %(survey[0]["exp_id"])
questions,required_count = parse_questions(survey_questions,
exp_id=survey[0]["exp_id"],
validate=True)
print "Testing validation generation of %s" %(survey[0]["exp_id"])
validation = parse_validation(required_count)
except:
if raise_error == False:
return False
raise
return True
def validate_surveys(survey_tags,
survey_repo,
survey_file="survey.tsv",
delim="\t",
raise_error=True):
'''validate_surveys validates an experiment factory survey folder
:param survey_tags: a list of surveys to validate
:param survey_repo: the survey repo with the survey folders to validate
:param survey_file: the default file to validate
:param raise_error: throw the error (meaning don't return True or False instead)
'''
if isinstance(survey_tags, str):
survey_tags = [survey_tags]
for survey_tag in survey_tags:
try:
survey_folder = "%s/%s" % (survey_repo, survey_tag)
print("Testing load of config.json for %s" % (survey_folder))
survey = load_experiment("%s" % survey_folder)
survey_questions = "%s/%s" % (survey_folder, survey_file)
print("Testing valid columns in %s" % (survey[0]["exp_id"]))
df = read_survey_file(survey_questions, delim=delim)
assert (isinstance(df, pandas.DataFrame))
print("Testing survey generation of %s" % (survey[0]["exp_id"]))
questions, required_count = parse_questions(
survey_questions, exp_id=survey[0]["exp_id"], validate=True)
print("Testing validation generation of %s" %
(survey[0]["exp_id"]))
validation = parse_validation(required_count)
except:
if raise_error == False:
return False
raise
return True
def validate_experiment_tag(experiment_folder):
'''validate_experiment_tag looks for definition of exp_id as the tag somewhere in experiment.js. We are only requiring one definition for now (a more lax approach), but this standard might be changed.
'''
experiments = find_directories(experiment_folder)
print "Testing %s experiment for definition of exp_id in experiment.js..."
for contender in experiments:
if validate(contender,warning=False) == True:
experiment = load_experiment(contender)
tag = experiment[0]["exp_id"]
# Experiment MUST contain experiment.js to run main experiment
print "TESTING %s for exp_id in experiment.js..." %tag
assert_equal("experiment.js" in experiment[0]["run"],True)
if "experiment.js" in experiment[0]["run"]:
experiment_js_file = open("%s/%s/experiment.js" %(experiment_folder,tag),"r")
experiment_js_list = [x.strip("\n").replace("'","").replace('"',"").replace(" ","") for x in experiment_js_file.readlines()]
experiment_js_file.close()
experiment_js = "".join(experiment_js_list)
[x]
has_exp_id = re.search("exp_id:%s" %tag,experiment_js) != None or re.search("exp_id=%s" %tag,experiment_js) != None
assert_equal(has_exp_id,True)
# Ensure all are formatted correctly
exp_id_instances = [re.findall("exp_id[=|:].+",x) for x in experiment_js_list if len(re.findall("exp_id[=|:].+,",x)) != 0]
line_numbers = [x+1 for x in range(len(experiment_js_list)) if len(re.findall("exp_id[=|:].+,",experiment_js_list[x])) != 0]
for e in range(len(exp_id_instances)):
exp_id_instance = exp_id_instances[e]
line_number = line_numbers[e]
print "Checking %s on line %s..." %(exp_id_instance[0],line_number)
assert_equal(re.search(tag,exp_id_instance[0])!=None,True)
def embed_experiment(folder,url_prefix=""):
'''embed_experiment
return an html snippet for embedding into an application. This assumes the same directory structure, that all jspsych files can be found in static/js/jspych, and experiments under static/experiments/[folder]
:param folder: full path to experiment folder with config.json
'''
folder = os.path.abspath(folder)
experiment = load_experiment("%s" %folder)
return get_experiment_html(experiment,folder,url_prefix=url_prefix)
def install_experiment_template(experiment, battery, router_url, to_dir):
'''install_experiment_template generates a survey or jspsych experiment template,
and is intended to be called from install_experiment_static, which takes care of other
dependencies like creating experiment folder, etc. The expfactory repos have validation and
testing before experiments are allowed to be submit, so we don't do extra checks.
:param experiment: the expdj.apps.experiments.models.Experiment
:param battery: the expdj.apps.experiments.models.Battery
:param router_url: the experiment router URL, to direct to on form submit
:param to_dir: the directory to install to, typically in a battery folder
'''
# If the experiment template is special (survey, experiment, game), we need to render the files
context = dict()
if experiment.template == "jspsych":
template = get_template("experiments/serve_battery.html")
config = load_experiment(to_dir)[0]
runcode = get_jspsych_init(config)
if experiment.template == "survey":
template = get_template("surveys/serve_battery.html")
config = load_experiment(to_dir)
runcode, validation = generate_survey(config,
to_dir,
form_action=router_url,
csrf_token=False)
context["validation"] = validation
# static path should be replaced with web path
url_path = "%s/" % (to_dir.replace(MEDIA_ROOT, MEDIA_URL[:-1]))
# prepare static files paths
css, js = prepare_header_scripts(config, url_prefix=url_path)
# Update the context dictionary, render the template
context["run"] = runcode
context["css"] = css
context["js"] = js
context["form_submit"] = router_url
context = Context(context)
# All templates will have the index written
return template.render(context)
def test_find_experiments(self):
# Test loading experiment folder path
experiments = get_experiments(self.battery_folder)
assert_equal(len(experiments), 1)
self.assertTrue(isinstance(experiments[0], str))
# Test loading experiment dictionary
config = get_experiments(self.battery_folder, load=True)
self.assertTrue(isinstance(config[0][0], dict))
loaded_experiment = load_experiment(self.experiment)
self.assertTrue(isinstance(loaded_experiment[0], dict))
def test_find_experiments(self):
# Test loading experiment folder path
experiments = get_experiments(self.battery_folder)
assert_equal(len(experiments),1)
self.assertTrue(isinstance(experiments[0],str))
# Test loading experiment dictionary
config = get_experiments(self.battery_folder,load=True)
self.assertTrue(isinstance(config[0][0],dict))
loaded_experiment = load_experiment(self.experiment)
self.assertTrue(isinstance(loaded_experiment[0],dict))
def test_experiment(folder=None,
battery_folder=None,
port=None,
pause_time=2000):
'''test_experiment
test an experiment locally with the --test tag and the expfactory robot
:param folder: full path to experiment folder to preview. If none specified, PWD is used
:param battery_folder: full path to battery folder to use as a template. If none specified, the expfactory-battery repo will be used.
:param port: the port number, default will be randomly generated between 8000 and 9999
'''
if folder == None:
folder = os.path.abspath(os.getcwd())
# Deploy experiment with battery to temporary directory
tmpdir = tmp_experiment(folder, battery_folder)
experiment = load_experiment("%s" % folder)
try:
if port == None:
port = choice(range(8000, 9999), 1)[0]
Handler = ExpfactoryServer
httpd = SocketServer.TCPServer(("", port), Handler)
server = Thread(target=httpd.serve_forever)
server.setDaemon(True)
server.start()
# Set up a web browser
browser = get_browser()
browser.implicitly_wait(3) # if error, will wait 3 seconds and retry
browser.set_page_load_timeout(10)
print "STARTING TEST OF EXPERIMENT %s" % (experiment[0]["exp_id"])
get_page(browser, "http://localhost:%s" % (port))
sleep(3)
count = 0
wait_time = 1000
while True:
print "Testing block %s of %s" % (count, experiment[0]["exp_id"])
wait_time, finished = test_block(browser, experiment, pause_time,
wait_time)
if finished == True:
break
count += 1
print "FINISHING TEST OF EXPERIMENT %s" % (experiment[0]["exp_id"])
except:
print "Stopping web server..."
httpd.server_close()
shutil.rmtree(tmpdir)
def _validate_folder(self, folder=None):
''' validate folder takes a cloned github repo, ensures
the existence of the config.json, and validates it.
'''
from expfactory.experiment import load_experiment
if folder is None:
folder = os.path.abspath(os.getcwd())
config = load_experiment(folder, return_path=True)
if not config:
return notvalid("%s is not an experiment." % (folder))
return self._validate_config(folder)
def _validate_folder(self, folder=None):
''' validate folder takes a cloned github repo, ensures
the existence of the config.json, and validates it.
'''
from expfactory.experiment import load_experiment
if folder is None:
folder=os.path.abspath(os.getcwd())
config = load_experiment(folder, return_path=True)
if not config:
return notvalid("%s is not an experiment." %(folder))
return self._validate_config(folder)
def get_experiment_run(valid_experiments,deployment="local"):
'''get_experiment_run
returns a dictionary of experiment run code (right now just jspsych init objects)
:param valid_experiments: full path to valid experiments folders, OR a loaded config.json (dict)
'''
runs = dict()
for valid_experiment in valid_experiments:
if not isinstance(valid_experiment,dict):
experiment = load_experiment(valid_experiment)[0]
else:
experiment = valid_experiment
tag = str(experiment["exp_id"])
if experiment["template"] == "jspsych":
runcode = get_jspsych_init(experiment,deployment=deployment)
runs[tag] = runcode
return runs
def get_experiment_run(valid_experiments, deployment="local"):
'''get_experiment_run
returns a dictionary of experiment run code (right now just jspsych init objects)
:param valid_experiments: full path to valid experiments folders, OR a loaded config.json (dict)
'''
runs = dict()
for valid_experiment in valid_experiments:
if not isinstance(valid_experiment, dict):
experiment = load_experiment(valid_experiment)[0]
else:
experiment = valid_experiment
tag = str(experiment["exp_id"])
if experiment["template"] == "jspsych":
runcode = get_jspsych_init(experiment, deployment=deployment)
runs[tag] = runcode
return runs
def test_experiment(folder=None,battery_folder=None,port=None,pause_time=2000):
'''test_experiment
test an experiment locally with the --test tag and the expfactory robot
:param folder: full path to experiment folder to preview. If none specified, PWD is used
:param battery_folder: full path to battery folder to use as a template. If none specified, the expfactory-battery repo will be used.
:param port: the port number, default will be randomly generated between 8000 and 9999
'''
if folder==None:
folder=os.path.abspath(os.getcwd())
# Deploy experiment with battery to temporary directory
tmpdir = tmp_experiment(folder,battery_folder)
experiment = load_experiment("%s" %folder)
try:
if port == None:
port = choice(range(8000,9999),1)[0]
Handler = ExpfactoryServer
httpd = SocketServer.TCPServer(("", port), Handler)
server = Thread(target=httpd.serve_forever)
server.setDaemon(True)
server.start()
# Set up a web browser
browser = get_browser()
browser.implicitly_wait(3) # if error, will wait 3 seconds and retry
browser.set_page_load_timeout(10)
print "STARTING TEST OF EXPERIMENT %s" %(experiment[0]["exp_id"])
get_page(browser,"http://localhost:%s" %(port))
sleep(3)
count=0
wait_time=1000
while True:
print "Testing block %s of %s" %(count,experiment[0]["exp_id"])
wait_time,finished = test_block(browser,experiment,pause_time,wait_time)
if finished == True:
break
count+=1
print "FINISHING TEST OF EXPERIMENT %s" %(experiment[0]["exp_id"])
except:
print "Stopping web server..."
httpd.server_close()
shutil.rmtree(tmpdir)
def get_load_static(valid_experiments,url_prefix="",unique=True):
'''get_load_static
return the scripts and styles as <link> and <script> to embed in a page directly
:param unique: return only unique scripts [default=True]
'''
loadstring = ""
for valid_experiment in valid_experiments:
experiment = load_experiment(valid_experiment)
css,js = get_stylejs(experiment,url_prefix=url_prefix)
loadstring = "%s%s%s" %(loadstring,js,css)
if unique == True:
scripts = loadstring.split("\n")
scripts_index = numpy.unique(scripts,return_index=True)[1]
# This ensures that scripts are loaded in same order as specified in config.json
unique_scripts = [scripts[idx] for idx in sorted(scripts_index)]
loadstring = "\n".join(unique_scripts)
return loadstring
def get_load_static(valid_experiments, url_prefix="", unique=True):
'''get_load_static
return the scripts and styles as <link> and <script> to embed in a page directly
:param unique: return only unique scripts [default=True]
'''
loadstring = ""
for valid_experiment in valid_experiments:
experiment = load_experiment(valid_experiment)
css, js = get_stylejs(experiment, url_prefix=url_prefix)
loadstring = "%s%s%s" % (loadstring, js, css)
if unique == True:
scripts = loadstring.split("\n")
scripts_index = list(np.unique(scripts, return_index=True))[1]
# This ensures that scripts are loaded in same order as specified in config.json
unique_scripts = [scripts[idx] for idx in sorted(scripts_index)]
loadstring = "\n".join(unique_scripts)
return loadstring
def get_timing_js(valid_experiments):
'''get_timing_js
Produce string (json / dictionary) of experiment timings
:param valid_experiments: a list of full paths to valid experiments to include
..note::
Produces the following format for each experiment
{name:"simple_rt", time: 3.5}, {name:"choice_rt", time: 4}, ...
'''
timingjs = []
for valid_experiment in valid_experiments:
experiment = load_experiment(valid_experiment)[0]
timingjs.append({"name":str(experiment["exp_id"]),"time":experiment["time"]})
return timingjs
def main():
parser = get_parser()
try:
args = parser.parse_args()
except:
sys.exit(0)
folder = args.folder
if folder is None:
folder = os.getcwd()
folder = os.path.abspath(folder)
survey = "%s/survey.tsv" % folder
if not os.path.exists(survey):
bot.error("Cannot find %s, required to generate survey." % survey)
sys.exit(1)
config = load_experiment(folder)
html, validation = generate_survey(config=config,
survey_file=survey,
form_action=args.action)
output = args.output
if output is None:
output = folder
output_index = "%s/index.html" % folder
if os.path.exists(output_index) and args.force is False:
bot.error("%s already exists, use --force to overwrite." %
output_index)
sys.exit(1)
bot.info("Writing output files to %s" % output_index)
template = get_template('survey/index.html')
template = sub_template(template, "{{html}}", html)
template = sub_template(template, "{{exp_id}}", config['exp_id'])
template = sub_template(template, "{{validation}}", validation)
write_file(output_index, template)
def tmp_experiment(folder=None, battery_folder=None):
'''generate temporary directory with experiment
:param folder: full path to experiment folder to preview (experiment, survey, or game). If none specified, PWD is used
:param battery_folder: full path to battery folder to use as a template. If none specified, the expfactory-battery repo will be used.
'''
if folder == None:
folder = os.path.abspath(os.getcwd())
if battery_folder == None:
tmpdir = custom_battery_download(repos=["battery"])
# If user has supplied a local battery folder, copy to tempdir
else:
tmpdir = tempfile.mkdtemp()
copy_directory(battery_folder, "%s/battery" % tmpdir)
experiment = load_experiment("%s" % folder)
tag = experiment[0]["exp_id"]
# Determine experiment template
experiment_type = "experiments"
if experiment[0]["template"] == "survey":
experiment_type = "surveys"
elif experiment[0]["template"] == "phaser":
experiment_type = "games"
# We will copy the entire experiment into the battery folder
battery_folder = "%s/battery" % (tmpdir)
experiment_folder = "%s/static/%s/%s" % (battery_folder, experiment_type,
tag)
if os.path.exists(experiment_folder):
shutil.rmtree(experiment_folder)
copy_directory(folder, experiment_folder)
index_file = "%s/index.html" % (battery_folder)
# Generate code for js and css
exp_template = get_experiment_html(experiment, experiment_folder)
filey = open(index_file, "w")
filey.writelines(exp_template)
filey.close()
os.chdir(battery_folder)
return tmpdir
def get_timing_js(valid_experiments):
'''get_timing_js
Produce string (json / dictionary) of experiment timings
:param valid_experiments: a list of full paths to valid experiments to include
..note::
Produces the following format for each experiment
{name:"simple_rt", time: 3.5}, {name:"choice_rt", time: 4}, ...
'''
timingjs = []
for valid_experiment in valid_experiments:
experiment = load_experiment(valid_experiment)[0]
timingjs.append({
"name": str(experiment["exp_id"]),
"time": experiment["time"]
})
return timingjs
def tmp_experiment(folder=None,battery_folder=None):
'''generate temporary directory with experiment
:param folder: full path to experiment folder to preview (experiment, survey, or game). If none specified, PWD is used
:param battery_folder: full path to battery folder to use as a template. If none specified, the expfactory-battery repo will be used.
'''
if folder==None:
folder=os.path.abspath(os.getcwd())
if battery_folder == None:
tmpdir = custom_battery_download(repos=["battery"])
# If user has supplied a local battery folder, copy to tempdir
else:
tmpdir = tempfile.mkdtemp()
copy_directory(battery_folder,"%s/battery" %tmpdir)
experiment = load_experiment("%s" %folder)
tag = experiment[0]["exp_id"]
# Determine experiment template
experiment_type = "experiments"
if experiment[0]["template"] == "survey":
experiment_type = "surveys"
elif experiment[0]["template"] == "phaser":
experiment_type = "games"
# We will copy the entire experiment into the battery folder
battery_folder = "%s/battery" %(tmpdir)
experiment_folder = "%s/static/%s/%s" %(battery_folder,experiment_type,tag)
if os.path.exists(experiment_folder):
shutil.rmtree(experiment_folder)
copy_directory(folder,experiment_folder)
index_file = "%s/index.html" %(battery_folder)
# Generate code for js and css
exp_template = get_experiment_html(experiment,experiment_folder)
filey = open(index_file,"w")
filey.writelines(exp_template)
filey.close()
os.chdir(battery_folder)
return tmpdir
def get_load_js(valid_experiments, url_prefix=""):
'''get_load_js
Return javascript to load list of valid experiments, based on psiturk.json
:param valid_experiments: a list of full paths to valid experiments to include
..note::
Format is:
{
case "simple_rt":
loadjscssfile("static/css/experiments/simple_rt.css","css")
loadjscssfile("static/js/experiments/simple_rt.js","js")
break;
case "choice_rt":
loadjscssfile("static/css/experiments/choice_rt.css","css")
loadjscssfile("static/js/experiments/choice_rt.js","js")
break;
...
}
'''
loadstring = "\n"
for valid_experiment in valid_experiments:
experiment = load_experiment(valid_experiment)[0]
tag = str(experiment["exp_id"])
loadstring = '%scase "%s":\n' % (loadstring, tag)
for script in experiment["run"]:
fname, ext = os.path.splitext(script)
ext = ext.replace(".", "").lower()
# If the file is included in the experiment
if len(script.split("/")) == 1:
loadstring = '%s loadjscssfile("%sstatic/experiments/%s/%s","%s")\n' % (
loadstring, url_prefix, tag, script, ext)
else:
loadstring = '%s loadjscssfile("%s%s","%s")\n' % (
loadstring, url_prefix, script, ext)
loadstring = "%s break;\n" % (loadstring)
return loadstring
def get_load_js(valid_experiments,url_prefix=""):
'''get_load_js
Return javascript to load list of valid experiments, based on psiturk.json
:param valid_experiments: a list of full paths to valid experiments to include
..note::
Format is:
{
case "simple_rt":
loadjscssfile("static/css/experiments/simple_rt.css","css")
loadjscssfile("static/js/experiments/simple_rt.js","js")
break;
case "choice_rt":
loadjscssfile("static/css/experiments/choice_rt.css","css")
loadjscssfile("static/js/experiments/choice_rt.js","js")
break;
...
}
'''
loadstring = "\n"
for valid_experiment in valid_experiments:
experiment = load_experiment(valid_experiment)[0]
tag = str(experiment["exp_id"])
loadstring = '%scase "%s":\n' %(loadstring,tag)
for script in experiment["run"]:
fname,ext = os.path.splitext(script)
ext = ext.replace(".","").lower()
# If the file is included in the experiment
if len(script.split("/")) == 1:
loadstring = '%s loadjscssfile("%sstatic/experiments/%s/%s","%s")\n' %(loadstring,url_prefix,tag,script,ext)
else:
loadstring = '%s loadjscssfile("%s%s","%s")\n' %(loadstring,url_prefix,script,ext)
loadstring = "%s break;\n" %(loadstring)
return loadstring
def main(args, parser, subparser):
# List of experiments is required
template = get_template('build/docker/Dockerfile.template')
# For now, only one database provided
database = args.database
studyid = args.studyid
experiments = args.experiments
template = sub_template(template, "{{studyid}}", studyid)
template = sub_template(template, "{{database}}", database)
library = get_library(key='name')
apps = "\n"
# Add local experiments to library, first preference
local_installs = 0
for experiment in experiments:
if os.path.exists(experiment):
bot.info('local experiment %s found, validating...' % experiment)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(experiment, cleanup=False)
if valid is True:
local_installs += 1
config = load_experiment(experiment)
exp_id = config['exp_id']
# If we aren't building in the experiment directory, we need to copy there
output_dir = "%s/%s" % (os.path.abspath(
os.path.dirname(args.output)), exp_id)
experiment_dir = os.path.abspath(experiment)
if output_dir != experiment_dir:
copy_directory(experiment_dir, output_dir)
config['local'] = os.path.abspath(experiment)
library[exp_id] = config
# Warn the user that local installs are not reproducible (from recipe)
if local_installs > 0:
bot.warning(
"%s local installs detected: build is not reproducible without experiment folders"
% local_installs)
# Build Image with Experiments
for experiment in experiments:
exp_id = os.path.basename(experiment)
if exp_id in library:
config = library[exp_id]
app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id)
# Here add custom build routine, should be list of lines
if "install" in config:
commands = "\n".join(
["RUN %s " % s for x in config['install']]).strip('\n')
app = "%s%s\n" % (app, commands)
# The final installation step, either from Github (url) or local folder
if "local" in config:
app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id)
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app, exp_id)
else:
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app, config['github'])
apps = "%s%s\n" % (apps, app)
else:
bot.warning('%s not in library, check spelling & punctuation.' %
exp_id)
if apps == "\n":
bot.error('No valid experiments found, cancelling recipe build.')
sys.exit(1)
template = sub_template(template, "{{experiments}}", apps)
outfile = write_file(args.output, template)
bot.log("Recipe written to %s" % outfile)
def test_getstylejs(self):
experiment = load_experiment(self.experiment)
stylejs = get_stylejs(experiment)
self.assertTrue(len(stylejs)==2)
self.assertTrue(re.search("style.css",stylejs[0])!=None)
self.assertTrue(re.search("experiment.js",stylejs[1])!=None)
def main():
parser = argparse.ArgumentParser(
description="generate experiments and infrastructure to serve them.")
parser.add_argument("--folder", dest='folder', help="full path to single experiment folder (for single experiment run with --run) or folder with many experiments (for battery run with --run)", type=str, default=None)
parser.add_argument("--subid", dest='subid', help="subject id to embed in experiments data in the case of a battery run with --run", type=str, default=None)
parser.add_argument("--experiments", dest='experiments', help="comma separated list of experiments for a local battery", type=str, default=None)
parser.add_argument("--port", dest='port', help="port to preview experiment", type=int, default=None)
parser.add_argument("--battery", dest='battery_folder', help="full path to local battery folder to use as template", type=str, default=None)
parser.add_argument("--time", dest='time', help="maximum number of minutes for battery to endure, to select experiments", type=int, default=99999)
parser.add_argument('--preview', help="preview an experiment locally (development function)", dest='preview', default=False, action='store_true')
parser.add_argument('--run', help="run a single experiment/survey or battery locally", dest='run', default=False, action='store_true')
parser.add_argument("--survey", dest='survey', help="survey to run for a local assessment", type=str, default=None)
parser.add_argument("--game", dest='game', help="game to run for a local assessment", type=str, default=None)
parser.add_argument('--validate', dest='validate', help="validate an experiment folder", default=False, action='store_true')
parser.add_argument('--psiturk', dest='psiturk', help="to be used with the --generate command, to generate a psiturk battery instead of local folder deployment", default=False, action='store_true')
parser.add_argument('--generate', dest='generate', help="generate (and don't run) a battery with --experiments to a --folder", default=False, action='store_true')
parser.add_argument("--output", dest='output', help="output folder for --generate command, if a temporary directory is not desired. Must not exist.", type=str, default=None)
parser.add_argument('--test', dest='test', help="test an experiment folder with the experiment robot", default=False, action='store_true')
try:
args = parser.parse_args()
except:
parser.print_help()
sys.exit(0)
# Check if the person wants to preview experiment or battery
if args.preview == True:
preview_experiment(folder=args.folder,battery_folder=args.battery_folder,port=args.port)
# Generate a local battery folder (static)
elif args.generate == True:
if args.experiments != None:
# Deploy a psiturk battery folder
experiments = args.experiments.split(",")
if args.psiturk == True:
outdir = generate(battery_dest=args.output,
battery_repo=args.battery_folder,
experiment_repo=args.folder,
experiments=experiments,
make_config=True,
warning=False)
# Deploy a regular battery folder
else:
outdir = generate_local(battery_dest=args.output,
subject_id="expfactory_battery_result",
battery_repo=args.battery_folder,
experiment_repo=args.folder,
experiments=experiments,
warning=False,
time=args.time)
print "Battery generation complete: static files are in %s" %(outdir)
else:
print "Please specify list of comma separated experiments with --experiments"
# Run a local battery
elif args.run == True:
# Warn the user about using repos for experiments and battery
if args.battery_folder == None:
print "No battery folder specified. Will pull latest battery from expfactory-battery repo"
if args.folder == None:
print "No experiments, games, or surveys folder specified. Will pull latest from expfactory-experiments repo"
if args.survey != None:
survey = args.survey.split(",")
if len(survey) > 0:
print "Currently only support running one survey, will run first in list."
survey = survey[0]
run_single(exp_id=survey,
repo_type="surveys",
source_repo=args.folder,
battery_repo=args.battery_folder,
port=args.port,
subject_id=args.subid)
if args.game != None:
game = args.game.split(",")
if len(game) > 0:
print "Currently only support running one game, will run first in list."
game = game[0]
run_single(exp_id=game,
repo_type="games",
source_repo=args.folder,
battery_repo=args.battery_folder,
port=args.port,
subject_id=args.subid)
if args.experiments != None:
experiments = args.experiments.split(",")
run_battery(experiments=experiments,
experiment_folder=args.folder,
subject_id=args.subid,
battery_folder=args.battery_folder,
port=args.port,
time=args.time)
else:
print "Please specify list of comma separated experiments with --experiments"
# Validate a config.json
elif args.validate == True:
if args.folder == None:
folder = os.getcwd()
validate(experiment_folder=folder)
# If a survey, and if validates, also validate survey.tsv
experiment = load_experiment(folder)[0]
if experiment["template"] == "survey":
print "Validating survey.tsv..."
survey_repo = os.path.dirname(folder)
validate_surveys(experiment["exp_id"],survey_repo)
# Run the experiment robot
elif args.test == True:
from expfactory.tests import test_experiment
test_experiment(folder=args.folder,battery_folder=args.battery_folder,port=args.port)
# Otherwise, just open the expfactory interface
else:
from expfactory.interface import start
start(port=args.port)
def main(args, parser, subparser):
folder = args.folder
if folder is None:
folder = os.getcwd()
source = args.src[0]
if source is None:
bot.error('Please provide a Github https address to install.')
sys.exit(1)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(source, cleanup=False)
exp_id = os.path.basename(source).replace('.git', '')
if valid is True:
# Local Install
if os.path.exists(source):
config = load_experiment(source)
source = os.path.abspath(source)
else:
config = load_experiment("%s/%s" % (cli.tmpdir, exp_id))
source = "%s/%s" % (cli.tmpdir, exp_id)
exp_id = config['exp_id']
python_module = exp_id.replace('-', '_').lower()
else:
bot.error('%s is not valid.' % exp_id)
sys.exit(1)
# Move static files to output folder
dest = "%s/%s" % (folder, exp_id)
bot.log("Installing %s to %s" % (exp_id, dest))
# Building container
in_container = False
if os.environ.get('SINGULARITY_IMAGE') is not None:
in_container = True
# Running, live container
elif os.environ.get('EXPFACTORY_CONTAINER') is not None:
in_container = True
if in_container is True:
# if in container, we always force
args.force = True
bot.log("Preparing experiment routes...")
template = get_template('experiments/template.py')
template = sub_template(template, '{{ exp_id }}', exp_id)
template = sub_template(template, '{{ exp_id_python }}', python_module)
# 1. Python blueprint
views = get_viewsdir(base=args.base)
view_output = "%s/%s.py" % (views, python_module)
save_template(view_output, template, base=views)
# 2. append to __init__
init = "%s/__init__.py" % views
with open(init, 'a') as filey:
filey.writelines('from .%s import *\n' % python_module)
# 3. Instructions
if "instructions" in config:
instruct = "%s/%s.help" % (views, python_module)
with open(instruct, 'w') as filey:
filey.writelines(config['instructions'])
if not os.path.exists(dest):
os.system('mkdir -p %s' % dest)
else:
if args.force is False:
bot.error('%s is not empty! Use --force to delete and re-create.' %
folder)
sys.exit(1)
# We don't need to copy if experiment already there
if source != dest:
os.system('cp -R %s/* %s' % (source, dest))
def main(args,parser,subparser):
template = 'build/docker/Dockerfile.template'
# Full path to template is required if provided via input
if args.input is not None:
template = args.input
template = get_template(template)
# For now, only one database provided
database = args.database
studyid = args.studyid
experiments = args.experiments
branch = "-b %s" %os.environ.get('EXPFACTORY_BRANCH','master')
headless = "false"
if args.headless is True:
headless = "true"
template = sub_template(template,"{{studyid}}",studyid)
template = sub_template(template,"{{database}}",database)
template = sub_template(template,"{{headless}}",headless)
template = sub_template(template,"{{branch}}",branch)
if args.headless is True:
bot.info("Headless build detected, you will need to generate tokens for application entry with expfactory users --new")
library = get_library(key='name')
apps = "\n"
# Add local experiments to library, first preference
local_installs = 0
for experiment in experiments:
if os.path.exists(experiment):
bot.info('local experiment %s found, validating...' %experiment)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(experiment, cleanup=False)
if valid is True:
local_installs +=1
config = load_experiment(experiment)
exp_id = config['exp_id']
# If we aren't building in the experiment directory, we need to copy there
output_dir = "%s/%s" %(os.path.abspath(os.path.dirname(args.output)), exp_id)
experiment_dir = os.path.abspath(experiment)
if output_dir != experiment_dir:
copy_directory(experiment_dir, output_dir)
config['local'] = os.path.abspath(experiment)
library[exp_id] = config
# Warn the user that local installs are not reproducible (from recipe)
if local_installs > 0:
bot.warning("%s local installs detected: build is not reproducible without experiment folders" %local_installs)
# Build Image with Experiments
for experiment in experiments:
exp_id = os.path.basename(experiment)
if exp_id in library:
config = library[exp_id]
app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" %(exp_id, exp_id)
# Here add custom build routine, should be list of lines
if "install" in config:
commands = "\n".join(["RUN %s "%s for x in config['install']]).strip('\n')
app = "%s%s\n" %(app, commands)
# The final installation step, either from Github (url) or local folder
if "local" in config:
app = "%sADD %s /scif/apps/%s\n" %(app, exp_id, exp_id)
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, exp_id)
else:
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, config['github'])
apps = "%s%s\n" %(apps,app)
else:
bot.warning('%s not in library, check spelling & punctuation.' %exp_id)
if apps == "\n":
bot.error('No valid experiments found, cancelling recipe build.')
sys.exit(1)
template = sub_template(template,"{{experiments}}",apps)
outfile = write_file(args.output,template)
bot.log("Recipe written to %s" %outfile)
def main(args, parser, subparser):
template = "build/docker/Dockerfile.template"
# Full path to template is required if provided via input
if args.input is not None:
template = args.input
template = get_template(template)
# For now, only one database provided
database = args.database
studyid = args.studyid
experiments = args.experiments
branch = "-b %s" % os.environ.get("EXPFACTORY_BRANCH", "master")
headless = "false"
if args.headless is True:
headless = "true"
template = sub_template(template, "{{studyid}}", studyid)
template = sub_template(template, "{{database}}", database)
template = sub_template(template, "{{headless}}", headless)
template = sub_template(template, "{{branch}}", branch)
if args.headless is True:
bot.info(
"Headless build detected, you will need to generate tokens for application entry with expfactory users --new"
)
library = get_library(key="name")
apps = "\n"
# Add local experiments to library, first preference
local_installs = 0
for experiment in experiments:
if os.path.exists(experiment):
bot.info("local experiment %s found, validating..." % experiment)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(experiment, cleanup=False)
if valid is True:
local_installs += 1
config = load_experiment(experiment)
exp_id = config["exp_id"]
# If we aren't building in the experiment directory, we need to copy there
output_dir = "%s/%s" % (
os.path.abspath(os.path.dirname(args.output)),
exp_id,
)
experiment_dir = os.path.abspath(experiment)
if output_dir != experiment_dir:
copy_directory(experiment_dir, output_dir)
config["local"] = os.path.abspath(experiment)
library[exp_id] = config
# Warn the user that local installs are not reproducible (from recipe)
if local_installs > 0:
bot.warning(
"%s local installs detected: build is not reproducible without experiment folders"
% local_installs
)
# Build Image with Experiments
for experiment in experiments:
exp_id = os.path.basename(experiment)
if exp_id in library:
config = library[exp_id]
app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id)
# Here add custom build routine, should be list of lines
if "install" in config:
commands = "\n".join(["RUN %s " % s for x in config["install"]]).strip(
"\n"
)
app = "%s%s\n" % (app, commands)
# The final installation step, either from Github (url) or local folder
if "local" in config:
app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id)
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app,
exp_id,
)
else:
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app,
config["github"],
)
apps = "%s%s\n" % (apps, app)
else:
bot.warning("%s not in library, check spelling & punctuation." % exp_id)
if apps == "\n":
bot.error("No valid experiments found, cancelling recipe build.")
sys.exit(1)
template = sub_template(template, "{{experiments}}", apps)
outfile = write_file(args.output, template)
bot.log("Recipe written to %s" % outfile)
def deploy_battery(deployment,
battery,
experiment_type,
context,
task_list,
template,
result,
next_page=None,
last_experiment=False):
'''deploy_battery is a general function for returning the final view to deploy a battery, either local or MTurk
:param deployment: either "docker-mturk" or "docker-local"
:param battery: models.Battery object
:param experiment_type: experiments,games,or surveys
:param context: context, which should already include next_page,
:param next_page: the next page to navigate to [optional] default is to reload the page to go to the next experiment
:param task_list: list of models.Experiment instances
:param template: html template to render
:param result: the result object, turk.models.Result
:param last_experiment: boolean if true will redirect the user to a page to submit the result (for surveys)
'''
if next_page == None:
next_page = "javascript:window.location.reload();"
context["next_page"] = next_page
# Check the user blacklist status
try:
blacklist = Blacklist.objects.get(worker=result.worker,
battery=battery)
if blacklist.active == True:
return render_to_response("experiments/blacklist.html")
except:
pass
# Get experiment folders
experiment_folders = [
os.path.join(media_dir, experiment_type, x.template.exp_id)
for x in task_list
]
context["experiment_load"] = get_load_static(experiment_folders,
url_prefix="/")
# Get code to run the experiment (not in external file)
runcode = ""
# Experiments templates
if experiment_type in ["experiments"]:
runcode = get_experiment_run(
experiment_folders,
deployment=deployment)[task_list[0].template.exp_id]
if result != None:
runcode = runcode.replace("{{result.id}}", str(result.id))
runcode = runcode.replace("{{next_page}}", next_page)
elif experiment_type in ["games"]:
experiment = load_experiment(experiment_folders[0])
runcode = experiment[0]["deployment_variables"]["run"]
elif experiment_type in ["surveys"]:
experiment = load_experiment(experiment_folders[0])
resultid = ""
if result != None:
resultid = result.id
runcode, validation = generate_survey(experiment,
experiment_folders[0],
form_action="/local/%s/" %
resultid,
csrf_token=True)
# Field will be filled in by browser cookie, and hidden fields are added for data
csrf_field = '<input type="hidden" name="csrfmiddlewaretoken" value="hello">'
csrf_field = '%s\n<input type="hidden" name="djstatus" value="FINISHED">' % (
csrf_field)
csrf_field = '%s\n<input type="hidden" name="url" value="chickenfingers">' % (
csrf_field)
runcode = runcode.replace("{% csrf_token %}", csrf_field)
context["validation"] = validation
if last_experiment == True:
context["last_experiment"] = last_experiment
context["run"] = runcode
response = render_to_response(template, context)
# without this header, the iFrame will not render in Amazon
response['x-frame-options'] = 'this_can_be_anything'
return response
def deploy_battery(deployment,battery,experiment_type,context,task_list,template,result,next_page=None,last_experiment=False):
'''deploy_battery is a general function for returning the final view to deploy a battery, either local or MTurk
:param deployment: either "docker-mturk" or "docker-local"
:param battery: models.Battery object
:param experiment_type: experiments,games,or surveys
:param context: context, which should already include next_page,
:param next_page: the next page to navigate to [optional] default is to reload the page to go to the next experiment
:param task_list: list of models.Experiment instances
:param template: html template to render
:param result: the result object, turk.models.Result
:param last_experiment: boolean if true will redirect the user to a page to submit the result (for surveys)
'''
if next_page == None:
next_page = "javascript:window.location.reload();"
context["next_page"] = next_page
# Check the user blacklist status
try:
blacklist = Blacklist.objects.get(worker=result.worker,battery=battery)
if blacklist.active == True:
return render_to_response("experiments/blacklist.html")
except:
pass
# Get experiment folders
experiment_folders = [os.path.join(media_dir,experiment_type,x.template.exp_id) for x in task_list]
context["experiment_load"] = get_load_static(experiment_folders,url_prefix="/")
# Get code to run the experiment (not in external file)
runcode = ""
# Experiments templates
if experiment_type in ["experiments"]:
runcode = get_experiment_run(experiment_folders,deployment=deployment)[task_list[0].template.exp_id]
if result != None:
runcode = runcode.replace("{{result.id}}",str(result.id))
runcode = runcode.replace("{{next_page}}",next_page)
elif experiment_type in ["games"]:
experiment = load_experiment(experiment_folders[0])
runcode = experiment[0]["deployment_variables"]["run"]
elif experiment_type in ["surveys"]:
experiment = load_experiment(experiment_folders[0])
resultid = ""
if result != None:
resultid = result.id
runcode,validation = generate_survey(experiment,experiment_folders[0],
form_action="/local/%s/" %resultid,
csrf_token=True)
# Field will be filled in by browser cookie, and hidden fields are added for data
csrf_field = '<input type="hidden" name="csrfmiddlewaretoken" value="hello">'
csrf_field = '%s\n<input type="hidden" name="djstatus" value="FINISHED">' %(csrf_field)
csrf_field = '%s\n<input type="hidden" name="url" value="chickenfingers">' %(csrf_field)
runcode = runcode.replace("{% csrf_token %}",csrf_field)
context["validation"] = validation
if last_experiment == True:
context["last_experiment"] = last_experiment
context["run"] = runcode
response = render_to_response(template, context)
# without this header, the iFrame will not render in Amazon
response['x-frame-options'] = 'this_can_be_anything'
return response
def main(args,parser,subparser):
folder = args.folder
if folder is None:
folder = os.getcwd()
source = args.src[0]
if source is None:
bot.error('Please provide a Github https address to install.')
sys.exit(1)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(source, cleanup=False)
exp_id = os.path.basename(source).replace('.git','')
if valid is True:
# Local Install
if os.path.exists(source):
config = load_experiment(source)
source = os.path.abspath(source)
else:
config = load_experiment("%s/%s" %(cli.tmpdir,exp_id))
source = "%s/%s" %(cli.tmpdir,exp_id)
exp_id = config['exp_id']
python_module = exp_id.replace('-','_').lower()
else:
bot.error('%s is not valid.' % exp_id)
sys.exit(1)
# Move static files to output folder
dest = "%s/%s" %(folder,exp_id)
bot.log("Installing %s to %s" %(exp_id, dest))
# Building container
in_container = False
if os.environ.get('SINGULARITY_IMAGE') is not None:
in_container = True
# Running, live container
elif os.environ.get('EXPFACTORY_CONTAINER') is not None:
in_container = True
if in_container is True:
# if in container, we always force
args.force = True
bot.log("Preparing experiment routes...")
template = get_template('experiments/template.py')
template = sub_template(template, '{{ exp_id }}', exp_id)
template = sub_template(template, '{{ exp_id_python }}', python_module)
# 1. Python blueprint
views = get_viewsdir(base=args.base)
view_output = "%s/%s.py" %(views, python_module)
save_template(view_output, template, base=views)
# 2. append to __init__
init = "%s/__init__.py" % views
with open(init,'a') as filey:
filey.writelines('from .%s import *\n' %python_module)
# 3. Instructions
if "instructions" in config:
instruct = "%s/%s.help" %(views, python_module)
with open(instruct,'w') as filey:
filey.writelines(config['instructions'])
if not os.path.exists(dest):
os.system('mkdir -p %s' %dest)
else:
if args.force is False:
bot.error('%s is not empty! Use --force to delete and re-create.' %folder)
sys.exit(1)
# We don't need to copy if experiment already there
if source != dest:
os.system('cp -R %s/* %s' %(source, dest))
