def validate_experiment_tag(experiment_folder):
'''validate_experiment_tag looks for definition of exp_id as the tag somewhere in experiment.js. We are only requiring one definition for now (a more lax approach), but this standard might be changed.
'''
experiments = find_directories(experiment_folder)
print "Testing %s experiment for definition of exp_id in experiment.js..."
for contender in experiments:
if validate(contender,warning=False) == True:
experiment = load_experiment(contender)
tag = experiment[0]["exp_id"]
# Experiment MUST contain experiment.js to run main experiment
print "TESTING %s for exp_id in experiment.js..." %tag
assert_equal("experiment.js" in experiment[0]["run"],True)
if "experiment.js" in experiment[0]["run"]:
experiment_js_file = open("%s/%s/experiment.js" %(experiment_folder,tag),"r")
experiment_js_list = [x.strip("\n").replace("'","").replace('"',"").replace(" ","") for x in experiment_js_file.readlines()]
experiment_js_file.close()
experiment_js = "".join(experiment_js_list)
[x]
has_exp_id = re.search("exp_id:%s" %tag,experiment_js) != None or re.search("exp_id=%s" %tag,experiment_js) != None
assert_equal(has_exp_id,True)
# Ensure all are formatted correctly
exp_id_instances = [re.findall("exp_id[=|:].+",x) for x in experiment_js_list if len(re.findall("exp_id[=|:].+,",x)) != 0]
line_numbers = [x+1 for x in range(len(experiment_js_list)) if len(re.findall("exp_id[=|:].+,",experiment_js_list[x])) != 0]
for e in range(len(exp_id_instances)):
exp_id_instance = exp_id_instances[e]
line_number = line_numbers[e]
print "Checking %s on line %s..." %(exp_id_instance[0],line_number)
assert_equal(re.search(tag,exp_id_instance[0])!=None,True)
def validate_experiment_tag(experiment_folder):
'''validate_experiment_tag looks for definition of exp_id as the tag somewhere in experiment.js. We are only requiring one definition for now (a more lax approach), but this standard might be changed.
'''
experiments = find_directories(experiment_folder)
print "Testing %s experiment for definition of exp_id in experiment.js..."
for contender in experiments:
if validate(contender,warning=False) == True:
experiment = load_experiment(contender)
tag = experiment[0]["exp_id"]
# Experiment MUST contain experiment.js to run main experiment
print "TESTING %s for exp_id in experiment.js..." %tag
assert_equal("experiment.js" in experiment[0]["run"],True)
if "experiment.js" in experiment[0]["run"]:
experiment_js_file = open("%s/%s/experiment.js" %(experiment_folder,tag),"r")
experiment_js_list = [x.strip("\n").replace("'","").replace('"',"").replace(" ","") for x in experiment_js_file.readlines()]
experiment_js_file.close()
experiment_js = "".join(experiment_js_list)
[x]
has_exp_id = re.search("exp_id:%s" %tag,experiment_js) != None or re.search("exp_id=%s" %tag,experiment_js) != None
assert_equal(has_exp_id,True)
# Ensure all are formatted correctly
exp_id_instances = [re.findall("exp_id[=|:].+",x) for x in experiment_js_list if len(re.findall("exp_id[=|:].+,",x)) != 0]
line_numbers = [x+1 for x in range(len(experiment_js_list)) if len(re.findall("exp_id[=|:].+,",experiment_js_list[x])) != 0]
for e in range(len(exp_id_instances)):
exp_id_instance = exp_id_instances[e]
line_number = line_numbers[e]
print "Checking %s on line %s..." %(exp_id_instance[0],line_number)
assert_equal(re.search(tag,exp_id_instance[0])!=None,True)
def get_experiments(experiment_repo,load=False,warning=True,repo_type="experiments"):
'''get_experiments
return loaded json for all valid experiments from an experiment folder
:param experiment_repo: full path to the experiments repo
:param load: if True, returns a list of loaded config.json objects. If False (default) returns the paths to the experiments
:param repo_type: tells the user what kind of task is being parsed, default is "experiments," but can also be "surveys" when called by get_surveys
'''
experiments = find_directories(experiment_repo)
valid_experiments = [e for e in experiments if validate(e,warning)]
print "Found %s valid %s" %(len(valid_experiments),repo_type)
if load == True:
valid_experiments = load_experiments(valid_experiments)
return valid_experiments
def get_experiments(base, load=False):
""" get_experiments will return loaded json for all valid experiments from an experiment folder
:param base: full path to the base folder with experiments inside
:param load: if True, returns a list of loaded config.json objects. If False (default) returns the paths to the experiments
"""
experiments = find_directories(base)
valid_experiments = [e for e in experiments if validate(e, cleanup=False)]
bot.info("Found %s valid experiments" % (len(valid_experiments)))
if load is True:
valid_experiments = load_experiments(valid_experiments)
# TODO at some point in this workflow we would want to grab instructions from help
# and variables from labels, environment, etc.
return valid_experiments
def get_experiments(base, load=False):
''' get_experiments will return loaded json for all valid experiments from an experiment folder
:param base: full path to the base folder with experiments inside
:param load: if True, returns a list of loaded config.json objects. If False (default) returns the paths to the experiments
'''
experiments = find_directories(base)
valid_experiments = [e for e in experiments if validate(e,cleanup=False)]
bot.info("Found %s valid experiments" %(len(valid_experiments)))
if load is True:
valid_experiments = load_experiments(valid_experiments)
#TODO at some point in this workflow we would want to grab instructions from help
# and variables from labels, environment, etc.
return valid_experiments
def validate_experiment_directories(experiment_folder):
experiments = find_directories(experiment_folder)
for contender in experiments:
assert_equal(validate(contender), True)
def validate_experiment_directories(experiment_folder):
experiments = find_directories(experiment_folder)
for contender in experiments:
assert_equal(validate(contender),True)
from expfactory.experiment import validate
from expfactory.utils import find_directories
myexperiments = find_directories("../expfactory-experiments")
for experiment in myexperiments:
if not validate(experiment):
print experiment
