def validate_experiment_tag(experiment_folder): '''validate_experiment_tag looks for definition of exp_id as the tag somewhere in experiment.js. We are only requiring one definition for now (a more lax approach), but this standard might be changed. ''' experiments = find_directories(experiment_folder) print "Testing %s experiment for definition of exp_id in experiment.js..." for contender in experiments: if validate(contender,warning=False) == True: experiment = load_experiment(contender) tag = experiment[0]["exp_id"] # Experiment MUST contain experiment.js to run main experiment print "TESTING %s for exp_id in experiment.js..." %tag assert_equal("experiment.js" in experiment[0]["run"],True) if "experiment.js" in experiment[0]["run"]: experiment_js_file = open("%s/%s/experiment.js" %(experiment_folder,tag),"r") experiment_js_list = [x.strip("\n").replace("'","").replace('"',"").replace(" ","") for x in experiment_js_file.readlines()] experiment_js_file.close() experiment_js = "".join(experiment_js_list) [x] has_exp_id = re.search("exp_id:%s" %tag,experiment_js) != None or re.search("exp_id=%s" %tag,experiment_js) != None assert_equal(has_exp_id,True) # Ensure all are formatted correctly exp_id_instances = [re.findall("exp_id[=|:].+",x) for x in experiment_js_list if len(re.findall("exp_id[=|:].+,",x)) != 0] line_numbers = [x+1 for x in range(len(experiment_js_list)) if len(re.findall("exp_id[=|:].+,",experiment_js_list[x])) != 0] for e in range(len(exp_id_instances)): exp_id_instance = exp_id_instances[e] line_number = line_numbers[e] print "Checking %s on line %s..." %(exp_id_instance[0],line_number) assert_equal(re.search(tag,exp_id_instance[0])!=None,True)
def get_experiments(experiment_repo,load=False,warning=True,repo_type="experiments"): '''get_experiments return loaded json for all valid experiments from an experiment folder :param experiment_repo: full path to the experiments repo :param load: if True, returns a list of loaded config.json objects. If False (default) returns the paths to the experiments :param repo_type: tells the user what kind of task is being parsed, default is "experiments," but can also be "surveys" when called by get_surveys ''' experiments = find_directories(experiment_repo) valid_experiments = [e for e in experiments if validate(e,warning)] print "Found %s valid %s" %(len(valid_experiments),repo_type) if load == True: valid_experiments = load_experiments(valid_experiments) return valid_experiments
def get_experiments(base, load=False): """ get_experiments will return loaded json for all valid experiments from an experiment folder :param base: full path to the base folder with experiments inside :param load: if True, returns a list of loaded config.json objects. If False (default) returns the paths to the experiments """ experiments = find_directories(base) valid_experiments = [e for e in experiments if validate(e, cleanup=False)] bot.info("Found %s valid experiments" % (len(valid_experiments))) if load is True: valid_experiments = load_experiments(valid_experiments) # TODO at some point in this workflow we would want to grab instructions from help # and variables from labels, environment, etc. return valid_experiments
def get_experiments(base, load=False): ''' get_experiments will return loaded json for all valid experiments from an experiment folder :param base: full path to the base folder with experiments inside :param load: if True, returns a list of loaded config.json objects. If False (default) returns the paths to the experiments ''' experiments = find_directories(base) valid_experiments = [e for e in experiments if validate(e,cleanup=False)] bot.info("Found %s valid experiments" %(len(valid_experiments))) if load is True: valid_experiments = load_experiments(valid_experiments) #TODO at some point in this workflow we would want to grab instructions from help # and variables from labels, environment, etc. return valid_experiments
def validate_experiment_directories(experiment_folder): experiments = find_directories(experiment_folder) for contender in experiments: assert_equal(validate(contender), True)
def validate_experiment_directories(experiment_folder): experiments = find_directories(experiment_folder) for contender in experiments: assert_equal(validate(contender),True)
from expfactory.experiment import validate from expfactory.utils import find_directories myexperiments = find_directories("../expfactory-experiments") for experiment in myexperiments: if not validate(experiment): print experiment