def test_validate(self):
validation = validate(self.experiment)
self.assertTrue(validation)
# Create temporary directory and test invalid config.json
broken_experiment = "%s/broken_experiment" %self.tmpdir
copy_directory(self.experiment,broken_experiment)
# Missing file specified
shutil.copyfile("%s/experiment.js" %broken_experiment,"%s/pizza.js" %broken_experiment)
os.remove("%s/experiment.js" %broken_experiment)
self.assertFalse(validate(broken_experiment))
shutil.copyfile("%s/pizza.js" %broken_experiment,"%s/experiment.js" %broken_experiment)
# Invalid field for template
broken_config = self.config[:]
broken_config[0]["template"] = "invalid_framework"
self.save_config(broken_config,broken_experiment)
self.assertFalse(validate(broken_experiment))
# Missing required field
broken_config = self.config[:]
del broken_config[0]["exp_id"]
self.save_config(broken_config,broken_experiment)
self.assertFalse(validate(broken_experiment))
def validate_experiment_tag(experiment_folder):
'''validate_experiment_tag looks for definition of exp_id as the tag somewhere in experiment.js. We are only requiring one definition for now (a more lax approach), but this standard might be changed.
'''
experiments = find_directories(experiment_folder)
print "Testing %s experiment for definition of exp_id in experiment.js..."
for contender in experiments:
if validate(contender,warning=False) == True:
experiment = load_experiment(contender)
tag = experiment[0]["exp_id"]
# Experiment MUST contain experiment.js to run main experiment
print "TESTING %s for exp_id in experiment.js..." %tag
assert_equal("experiment.js" in experiment[0]["run"],True)
if "experiment.js" in experiment[0]["run"]:
experiment_js_file = open("%s/%s/experiment.js" %(experiment_folder,tag),"r")
experiment_js_list = [x.strip("\n").replace("'","").replace('"',"").replace(" ","") for x in experiment_js_file.readlines()]
experiment_js_file.close()
experiment_js = "".join(experiment_js_list)
[x]
has_exp_id = re.search("exp_id:%s" %tag,experiment_js) != None or re.search("exp_id=%s" %tag,experiment_js) != None
assert_equal(has_exp_id,True)
# Ensure all are formatted correctly
exp_id_instances = [re.findall("exp_id[=|:].+",x) for x in experiment_js_list if len(re.findall("exp_id[=|:].+,",x)) != 0]
line_numbers = [x+1 for x in range(len(experiment_js_list)) if len(re.findall("exp_id[=|:].+,",experiment_js_list[x])) != 0]
for e in range(len(exp_id_instances)):
exp_id_instance = exp_id_instances[e]
line_number = line_numbers[e]
print "Checking %s on line %s..." %(exp_id_instance[0],line_number)
assert_equal(re.search(tag,exp_id_instance[0])!=None,True)
def validate_experiment_directories(experiment_folder):
experiments = find_directories(experiment_folder)
for contender in experiments:
assert_equal(validate(contender),True)
def validate_experiment_directories(experiment_folder):
experiments = find_directories(experiment_folder)
for contender in experiments:
assert (validate(contender))
def main():
parser = argparse.ArgumentParser(
description="generate experiments and infrastructure to serve them.")
parser.add_argument("--folder", dest='folder', help="full path to single experiment folder (for single experiment run with --run) or folder with many experiments (for battery run with --run)", type=str, default=None)
parser.add_argument("--subid", dest='subid', help="subject id to embed in experiments data in the case of a battery run with --run", type=str, default=None)
parser.add_argument("--experiments", dest='experiments', help="comma separated list of experiments for a local battery", type=str, default=None)
parser.add_argument("--port", dest='port', help="port to preview experiment", type=int, default=None)
parser.add_argument("--battery", dest='battery_folder', help="full path to local battery folder to use as template", type=str, default=None)
parser.add_argument("--time", dest='time', help="maximum number of minutes for battery to endure, to select experiments", type=int, default=99999)
parser.add_argument('--preview', help="preview an experiment locally (development function)", dest='preview', default=False, action='store_true')
parser.add_argument('--run', help="run a single experiment/survey or battery locally", dest='run', default=False, action='store_true')
parser.add_argument("--survey", dest='survey', help="survey to run for a local assessment", type=str, default=None)
parser.add_argument("--game", dest='game', help="game to run for a local assessment", type=str, default=None)
parser.add_argument('--validate', dest='validate', help="validate an experiment folder", default=False, action='store_true')
parser.add_argument('--psiturk', dest='psiturk', help="to be used with the --generate command, to generate a psiturk battery instead of local folder deployment", default=False, action='store_true')
parser.add_argument('--generate', dest='generate', help="generate (and don't run) a battery with --experiments to a --folder", default=False, action='store_true')
parser.add_argument("--output", dest='output', help="output folder for --generate command, if a temporary directory is not desired. Must not exist.", type=str, default=None)
parser.add_argument('--test', dest='test', help="test an experiment folder with the experiment robot", default=False, action='store_true')
try:
args = parser.parse_args()
except:
parser.print_help()
sys.exit(0)
# Check if the person wants to preview experiment or battery
if args.preview == True:
preview_experiment(folder=args.folder,battery_folder=args.battery_folder,port=args.port)
# Generate a local battery folder (static)
elif args.generate == True:
if args.experiments != None:
# Deploy a psiturk battery folder
experiments = args.experiments.split(",")
if args.psiturk == True:
outdir = generate(battery_dest=args.output,
battery_repo=args.battery_folder,
experiment_repo=args.folder,
experiments=experiments,
make_config=True,
warning=False)
# Deploy a regular battery folder
else:
outdir = generate_local(battery_dest=args.output,
subject_id="expfactory_battery_result",
battery_repo=args.battery_folder,
experiment_repo=args.folder,
experiments=experiments,
warning=False,
time=args.time)
print "Battery generation complete: static files are in %s" %(outdir)
else:
print "Please specify list of comma separated experiments with --experiments"
# Run a local battery
elif args.run == True:
# Warn the user about using repos for experiments and battery
if args.battery_folder == None:
print "No battery folder specified. Will pull latest battery from expfactory-battery repo"
if args.folder == None:
print "No experiments, games, or surveys folder specified. Will pull latest from expfactory-experiments repo"
if args.survey != None:
survey = args.survey.split(",")
if len(survey) > 0:
print "Currently only support running one survey, will run first in list."
survey = survey[0]
run_single(exp_id=survey,
repo_type="surveys",
source_repo=args.folder,
battery_repo=args.battery_folder,
port=args.port,
subject_id=args.subid)
if args.game != None:
game = args.game.split(",")
if len(game) > 0:
print "Currently only support running one game, will run first in list."
game = game[0]
run_single(exp_id=game,
repo_type="games",
source_repo=args.folder,
battery_repo=args.battery_folder,
port=args.port,
subject_id=args.subid)
if args.experiments != None:
experiments = args.experiments.split(",")
run_battery(experiments=experiments,
experiment_folder=args.folder,
subject_id=args.subid,
battery_folder=args.battery_folder,
port=args.port,
time=args.time)
else:
print "Please specify list of comma separated experiments with --experiments"
# Validate a config.json
elif args.validate == True:
if args.folder == None:
folder = os.getcwd()
validate(experiment_folder=folder)
# If a survey, and if validates, also validate survey.tsv
experiment = load_experiment(folder)[0]
if experiment["template"] == "survey":
print "Validating survey.tsv..."
survey_repo = os.path.dirname(folder)
validate_surveys(experiment["exp_id"],survey_repo)
# Run the experiment robot
elif args.test == True:
from expfactory.tests import test_experiment
test_experiment(folder=args.folder,battery_folder=args.battery_folder,port=args.port)
# Otherwise, just open the expfactory interface
else:
from expfactory.interface import start
start(port=args.port)
from expfactory.experiment import validate
from expfactory.utils import find_directories
myexperiments = find_directories("../expfactory-experiments")
for experiment in myexperiments:
if not validate(experiment):
print experiment
