def test_find_experiments(self): # Test loading experiment folder path experiments = get_experiments(self.battery_folder) assert_equal(len(experiments), 1) self.assertTrue(isinstance(experiments[0], str)) # Test loading experiment dictionary config = get_experiments(self.battery_folder, load=True) self.assertTrue(isinstance(config[0][0], dict)) loaded_experiment = load_experiment(self.experiment) self.assertTrue(isinstance(loaded_experiment[0], dict))
def test_find_experiments(self): # Test loading experiment folder path experiments = get_experiments(self.battery_folder) assert_equal(len(experiments),1) self.assertTrue(isinstance(experiments[0],str)) # Test loading experiment dictionary config = get_experiments(self.battery_folder,load=True) self.assertTrue(isinstance(config[0][0],dict)) loaded_experiment = load_experiment(self.experiment) self.assertTrue(isinstance(loaded_experiment[0],dict))
def generate_base(battery_dest,tasks=None,experiment_repo=None,survey_repo=None,game_repo=None, add_experiments=True,add_surveys=True,add_games=True,battery_repo=None,warning=True): '''generate_base returns a folder with downloaded experiments, surveys, and battery, either specified by the user or a temporary directory, to be used by generate_local and generate (for psiturk) :param battery_dest: [required] is the output folder for your battery. This folder MUST NOT EXIST. :param battery_repo: location of psiturk-battery repo to use as a template. If not specified, will be downloaded to a temporary directory :param experiment_repo: location of a expfactory-experiments repo to check for valid experiments. If not specified, will be downloaded to a temporary directory :param survey_repo: location of a expfactory-surveys repo to check for valid surveys. If not specified, will be downloaded to a temporary directory :param tasks: a list of experiments and surveys, meaning the "exp_id" variable in the config.json, to include. This variable also conincides with the tasks folder name. :param warning: show warnings when validating experiments (default True) ''' if experiment_repo == None or battery_repo == None or survey_repo == None or game_repo == None: tmpdir = custom_battery_download() if experiment_repo == None: experiment_repo = "%s/experiments" %(tmpdir) if battery_repo == None: battery_repo = "%s/battery" %(tmpdir) if survey_repo == None: survey_repo = "%s/surveys" %(tmpdir) if game_repo == None: game_repo = "%s/games" %(tmpdir) # Copy battery skeleton to destination copy_directory(battery_repo,battery_dest) valid_experiments = [] valid_surveys = [] valid_games = [] if add_experiments == True: valid_experiments = get_experiments(experiment_repo,warning=warning) if add_surveys == True: valid_surveys = get_experiments(survey_repo,warning=warning,repo_type="surveys") if add_games == True: valid_games = get_experiments(game_repo,warning=warning,repo_type="games") # If the user wants to select a subset if tasks != None: valid_experiments = [x for x in valid_experiments if os.path.basename(x) in [os.path.basename(e) for e in tasks]] valid_surveys = [x for x in valid_surveys if os.path.basename(x) in [os.path.basename(e) for e in tasks]] valid_games = [x for x in valid_games if os.path.basename(x) in [os.path.basename(e) for e in tasks]] base = {"battery_repo":battery_repo, "experiment_repo":experiment_repo, "survey_repo":survey_repo, "game_repo":game_repo, "experiments":valid_experiments, "surveys":valid_surveys, "games":valid_games} return base
def get_surveys(survey_repo=None,load=False,warning=True,repo_type="surveys"): '''get_surveys is a wrapper for "get_experiments" - the functionality is the same, but provided for users: return loaded json for all valid survyes from an surveys folder :param survey_repo: full path to the surveys repo :param load: if True, returns a list of loaded config.json objects. If False (default) returns the paths to the experiments :param repo_type: tells the user what kind of task is being parsed, default is "experiments," but can also be "surveys" when called by get_surveys ''' return get_experiments(experiment_repo=survey_repo,load=load,warning=warning,repo_type=repo_type)
def setup(self): ''' obtain database and filesystem preferences from defaults, and compare with selection in container. ''' self.selection = EXPFACTORY_EXPERIMENTS self.ordered = len(EXPFACTORY_EXPERIMENTS) > 0 self.data_base = EXPFACTORY_DATA self.study_id = EXPFACTORY_SUBID self.base = EXPFACTORY_BASE self.randomize = EXPFACTORY_RANDOMIZE self.headless = EXPFACTORY_HEADLESS # Generate variables, if they exist self.vars = generate_runtime_vars() or None available = get_experiments("%s" % self.base) self.experiments = get_selection(available, self.selection) self.logger.debug(self.experiments) self.lookup = make_lookup(self.experiments) final = "\n".join(list(self.lookup.keys())) bot.log("Headless mode: %s" % self.headless) bot.log("User has selected: %s" % self.selection) bot.log("Experiments Available: %s" %"\n".join(available)) bot.log("Randomize: %s" % self.randomize) bot.log("Final Set \n%s" % final)
def setup(self): """ obtain database and filesystem preferences from defaults, and compare with selection in container. """ self.selection = EXPFACTORY_EXPERIMENTS self.ordered = len(EXPFACTORY_EXPERIMENTS) > 0 self.data_base = EXPFACTORY_DATA self.study_id = EXPFACTORY_SUBID self.base = EXPFACTORY_BASE self.randomize = EXPFACTORY_RANDOMIZE self.headless = EXPFACTORY_HEADLESS # Generate variables, if they exist self.vars = generate_runtime_vars() or None available = get_experiments("%s" % self.base) self.experiments = get_selection(available, self.selection) self.logger.debug(self.experiments) self.lookup = make_lookup(self.experiments) final = "\n".join(list(self.lookup.keys())) bot.log("Headless mode: %s" % self.headless) bot.log("User has selected: %s" % self.selection) bot.log("Experiments Available: %s" % "\n".join(available)) bot.log("Randomize: %s" % self.randomize) bot.log("Final Set \n%s" % final)
def validate_circle_yml(experiment_repo, repo_type="experiments"): '''validate_circle_yml will make sure that all experiment folders in a repo are tested with circle_ci_test in the circle.yml :param experiment_repo: the experiment repo folder that contains experiment subdirectories with config.json files ''' if "CIRCLE_BRANCH" in os.environ.keys(): circle_yml_file = "%s/circle.yml" % experiment_repo assert_equal(os.path.exists(circle_yml_file), True) circle_yml_file = open(circle_yml_file, "r") circle_yml = "".join([ x.strip("\n").replace(" ", "").replace("'", "").replace('"', "") for x in circle_yml_file.readlines() ]) circle_yml = circle_yml.replace("(", "").replace(")", "") experiments = get_experiments(experiment_repo, load=True, warning=False, repo_type=repo_type) tags = [x[0]["exp_id"] for x in experiments] for tag in tags: print "TESTING if %s defined for circle ci testing in circle.yml..." % tag if repo_type == "experiments": assert_equal( re.search("circle_ci_test%s" % tag, circle_yml) != None, True) elif repo_type == "surveys": assert_equal( re.search("circle_ci_survey%s" % tag, circle_yml) != None, True) else: print "Not in a continuous integration (CircleCI) environment, skipping test." print "All experiments found in circle.yml for testing!"
def experiment_robot_web(web_folder,experiment_tags=None,port=None,pause_time=100): '''experiment_robot_web Robot to automatically run and test experiments, to work with an experiment web folder (meaning produced with views.get_experiment_web. This folder has the standard battery structure with experiment pre-generated as html files. A separate function will/should eventually be made for single experiment preview. :param experiment_tags: experimentweb generated folder with generate_experiment_web :param experiment_tags: list of experiment folders to test :param pause_time: time to wait between tasks, in addition to time specified in jspsych :param port: port. Randomly selected if None is selected ''' experimentweb_base = os.path.abspath(web_folder) if port == None: port = choice(range(8000,9999),1)[0] Handler = ExpfactoryServer httpd = SocketServer.TCPServer(("", port), Handler) server = Thread(target=httpd.serve_forever) server.setDaemon(True) server.start() # Set up a web browser os.chdir(experimentweb_base) browser = get_browser() browser.implicitly_wait(3) # if error, will wait 3 seconds and retry browser.set_page_load_timeout(10) # Find experiments experiments = get_experiments("%s/static/experiments" %experimentweb_base,load=True,warning=False) if experiment_tags != None: experiments = [e for e in experiments if e[0]["exp_id"] in experiment_tags] print "Found %s experiments to test." %(len(experiments)) for experiment in experiments: print "STARTING TEST OF EXPERIMENT %s" %(experiment[0]["exp_id"]) get_page(browser,"http://localhost:%s/%s.html" %(port,experiment[0]["exp_id"])) sleep(3) count=0 wait_time=0 while True: print "Testing command %s of %s" %(count,experiment[0]["exp_id"]) try: wait_time,finished = test_block(browser,experiment,pause_time,wait_time) if finished == True: break count+=1 except UnexpectedAlertPresentException: print "Found alert: closing." try: alert = browser.switch_to_alert() alert.accept() except: pass print "FINISHING TEST OF EXPERIMENT %s" %(experiment[0]["exp_id"]) # Stop the server httpd.server_close()
def install_experiments(experiment_tags=None, repo_type="experiments"): # We will return list of experiments that did not install successfully errored_experiments = [] tmpdir = custom_battery_download(repos=[repo_type, "battery"]) # The git commit is saved with the experiment as the "version" repo = Repo("%s/%s" % (tmpdir, repo_type)) commit = repo.commit("master").__str__() experiments = get_experiments("%s/%s" % (tmpdir, repo_type), load=True, warning=False) if experiment_tags != None: experiments = [e for e in experiments if e[0]["exp_id"] in experiment_tags] for experiment in experiments: try: performance_variable = None rejection_variable = None if "experiment_variables" in experiment[0]: if isinstance(experiment[0]["experiment_variables"], list): for var in experiment[0]["experiment_variables"]: if var["type"].lower().strip() == "bonus": performance_variable = parse_experiment_variable(var) elif var["type"].lower().strip() == "credit": rejection_variable = parse_experiment_variable(var) else: parse_experiment_variable(var) # adds to database if isinstance(experiment[0]["reference"], list): reference = experiment[0]["reference"][0] else: reference = experiment[0]["reference"] cognitive_atlas_task = get_cognitiveatlas_task(experiment[0]["cognitive_atlas_task_id"]) new_experiment, _ = ExperimentTemplate.objects.update_or_create( exp_id=experiment[0]["exp_id"], defaults={ "name": experiment[0]["name"], "cognitive_atlas_task": cognitive_atlas_task, "publish": bool(experiment[0]["publish"]), "time": experiment[0]["time"], "reference": reference, "version": commit, "template": experiment[0]["template"], "performance_variable": performance_variable, "rejection_variable": rejection_variable, }, ) new_experiment.save() experiment_folder = "%s/%s/%s" % (tmpdir, repo_type, experiment[0]["exp_id"]) output_folder = "%s/%s/%s" % (media_dir, repo_type, experiment[0]["exp_id"]) if os.path.exists(output_folder): shutil.rmtree(output_folder) copy_directory(experiment_folder, output_folder) except: errored_experiments.append(experiment[0]["exp_id"]) shutil.rmtree(tmpdir) return errored_experiments
def __init__(self, *args, **kwargs): super(EFServer, self).__init__(*args, **kwargs) # download repo on start of application self.tmpdir = tempfile.mkdtemp() custom_battery_download(tmpdir=self.tmpdir) self.experiments = get_experiments("%s/experiments" %self.tmpdir,load=True,warning=False) self.experiment_lookup = make_lookup(self.experiments,"exp_id")
def test_generate(self): # Test loading experiment folder path generate(self.battery) self.assertTrue(os.path.exists(self.battery)) # Test loading experiments experiments = get_experiments("%s/static/experiments" %self.battery,load=True) self.assertTrue(len(experiments)>0)
def survey_robot_web(web_folder, survey_tags=None, port=None, pause_time=100): '''survey_robot_web Robot to automatically click through surveys, to work with an experiment web folder (meaning produced with views.get_experiment_web. This folder has the standard battery structure with experiment pre-generated as html files. :param web_folder: experimentweb generated folder with generate_experiment_web :param survey_tags: list of experiment folders to test :param pause_time: time to wait between tasks, in addition to time specified in jspsych :param port: port. Randomly selected if None is selected ''' web_base = os.path.abspath(web_folder) httpd, port = get_web_server(port=port) # Set up a web browser os.chdir(web_base) browser = get_browser() browser.implicitly_wait(3) # if error, will wait 3 seconds and retry browser.set_page_load_timeout(10) # Find surveys surveys = get_experiments("%s/static/surveys" % web_base, load=True, warning=False, repo_type="surveys") if survey_tags != None: surveys = [s for s in surveys if s[0]["exp_id"] in survey_tags] print "Found %s surveys to test." % (len(surveys)) for survey in surveys: print "STARTING TEST OF SURVEY %s" % (survey[0]["exp_id"]) get_page(browser, "http://localhost:%s/%s.html" % (port, survey[0]["exp_id"])) sleep(3) count = 1 while True: print "Testing page %s of %s" % (count, survey[0]["exp_id"]) try: finished = advance_survey(browser, pause_time) if finished == True: break count += 1 except UnexpectedAlertPresentException: print "Found alert: closing." try: alert = browser.switch_to_alert() alert.accept() except: pass print "FINISHING TEST OF SURVEY %s" % (survey[0]["exp_id"]) # Stop the server httpd.server_close()
def get_experiment_selection(repo_type="experiments"): # tmpdir = custom_battery_download(repos=[repo_type]) tmpdir = os.path.join(EXP_REPO, 'expfactory-{}'.format(repo_type)) experiments = get_experiments(tmpdir, load=True, warning=False, repo_type=repo_type) experiments = [x[0] for x in experiments] # shutil.rmtree(tmpdir) return experiments
def test_generate(self): # Test loading experiment folder path generate(self.battery) self.assertTrue(os.path.exists(self.battery)) # Test loading experiments experiments = get_experiments("%s/static/experiments" % self.battery, load=True) self.assertTrue(len(experiments) > 0)
def install_experiments(experiment_tags=None,repo_type="experiments"): # We will return list of experiments that did not install successfully errored_experiments = [] tmpdir = custom_battery_download(repos=[repo_type,"battery"]) # The git commit is saved with the experiment as the "version" repo = Repo("%s/%s" %(tmpdir,repo_type)) commit = repo.commit('master').__str__() experiments = get_experiments("%s/%s" %(tmpdir,repo_type),load=True,warning=False) if experiment_tags != None: experiments = [e for e in experiments if e[0]["exp_id"] in experiment_tags] for experiment in experiments: try: performance_variable = None rejection_variable = None if "experiment_variables" in experiment[0]: if isinstance(experiment[0]["experiment_variables"],list): for var in experiment[0]["experiment_variables"]: if var["type"].lower().strip() == "bonus": performance_variable = parse_experiment_variable(var) elif var["type"].lower().strip() == "credit": rejection_variable = parse_experiment_variable(var) else: parse_experiment_variable(var) # adds to database if isinstance(experiment[0]["reference"],list): reference = experiment[0]["reference"][0] else: reference = experiment[0]["reference"] cognitive_atlas_task = get_cognitiveatlas_task(experiment[0]["cognitive_atlas_task_id"]) new_experiment,_ = ExperimentTemplate.objects.update_or_create(exp_id=experiment[0]["exp_id"], defaults={"name":experiment[0]["name"], "cognitive_atlas_task":cognitive_atlas_task, "publish":bool(experiment[0]["publish"]), "time":experiment[0]["time"], "reference":reference, "version":commit, "template":experiment[0]["template"], "performance_variable":performance_variable, "rejection_variable":rejection_variable}) new_experiment.save() experiment_folder = "%s/%s/%s" %(tmpdir,repo_type,experiment[0]["exp_id"]) output_folder = "%s/%s/%s" %(media_dir,repo_type,experiment[0]["exp_id"]) if os.path.exists(output_folder): shutil.rmtree(output_folder) copy_directory(experiment_folder,output_folder) except: errored_experiments.append(experiment[0]["exp_id"]) shutil.rmtree(tmpdir) return errored_experiments
def experiment_robot_web(web_folder,experiment_tags=None,port=None,pause_time=100): '''experiment_robot_web Robot to automatically run and test experiments, to work with an experiment web folder (meaning produced with views.get_experiment_web. This folder has the standard battery structure with experiment pre-generated as html files. :param web_folder: experimentweb generated folder with generate_experiment_web :param experiment_tags: list of experiment folders to test :param pause_time: time to wait between tasks, in addition to time specified in jspsych :param port: port. Randomly selected if None is selected ''' experimentweb_base = os.path.abspath(web_folder) httpd,port = get_web_server(port=port) # Set up a web browser os.chdir(experimentweb_base) browser = get_browser() browser.implicitly_wait(3) # if error, will wait 3 seconds and retry browser.set_page_load_timeout(10) # Find experiments experiments = get_experiments("%s/static/experiments" %experimentweb_base,load=True,warning=False) if experiment_tags != None: experiments = [e for e in experiments if e[0]["exp_id"] in experiment_tags] print "Found %s experiments to test." %(len(experiments)) for experiment in experiments: print "STARTING TEST OF EXPERIMENT %s" %(experiment[0]["exp_id"]) get_page(browser,"http://localhost:%s/%s.html" %(port,experiment[0]["exp_id"])) sleep(3) count=0 wait_time=0 while True: print "Testing command %s of %s" %(count,experiment[0]["exp_id"]) try: wait_time,finished = test_block(browser,experiment,pause_time,wait_time) if finished == True: break count+=1 except UnexpectedAlertPresentException: print "Found alert: closing." try: alert = browser.switch_to_alert() alert.accept() except: pass print "FINISHING TEST OF EXPERIMENT %s" %(experiment[0]["exp_id"]) # Stop the server httpd.server_close()
def survey_robot_web(web_folder,survey_tags=None,port=None,pause_time=100): '''survey_robot_web Robot to automatically click through surveys, to work with an experiment web folder (meaning produced with views.get_experiment_web. This folder has the standard battery structure with experiment pre-generated as html files. :param web_folder: experimentweb generated folder with generate_experiment_web :param survey_tags: list of experiment folders to test :param pause_time: time to wait between tasks, in addition to time specified in jspsych :param port: port. Randomly selected if None is selected ''' web_base = os.path.abspath(web_folder) httpd,port = get_web_server(port=port) # Set up a web browser os.chdir(web_base) browser = get_browser() browser.implicitly_wait(3) # if error, will wait 3 seconds and retry browser.set_page_load_timeout(10) # Find surveys surveys = get_experiments("%s/static/surveys" %web_base,load=True,warning=False,repo_type="surveys") if survey_tags != None: surveys = [s for s in surveys if s[0]["exp_id"] in survey_tags] print "Found %s surveys to test." %(len(surveys)) for survey in surveys: print "STARTING TEST OF SURVEY %s" %(survey[0]["exp_id"]) get_page(browser,"http://localhost:%s/%s.html" %(port,survey[0]["exp_id"])) sleep(3) count=1 while True: print "Testing page %s of %s" %(count,survey[0]["exp_id"]) try: finished = advance_survey(browser,pause_time) if finished == True: break count+=1 except UnexpectedAlertPresentException: print "Found alert: closing." try: alert = browser.switch_to_alert() alert.accept() except: pass print "FINISHING TEST OF SURVEY %s" %(survey[0]["exp_id"]) # Stop the server httpd.server_close()
def get_surveys(survey_repo=None, load=False, warning=True, repo_type="surveys"): '''get_surveys is a wrapper for "get_experiments" - the functionality is the same, but provided for users: return loaded json for all valid survyes from an surveys folder :param survey_repo: full path to the surveys repo :param load: if True, returns a list of loaded config.json objects. If False (default) returns the paths to the experiments :param repo_type: tells the user what kind of task is being parsed, default is "experiments," but can also be "surveys" when called by get_surveys ''' return get_experiments(experiment_repo=survey_repo, load=load, warning=warning, repo_type=repo_type)
def get_cognitiveatlas_hierarchy(experiment_tags=None,get_html=False): '''get_cognitiveatlas_hierarchy return :param experiment_tags: a list of experiment exp_id tags to include. If None provided, all valid experiments will be used. :param get_html: if True, returns rendered HTML template with hierarchy. False returns json data structure. ''' from cognitiveatlas.datastructure import concept_node_triples from expfactory.graph import make_tree_from_triples tmpdir = custom_battery_download() experiment_folder = "%s/experiments" %tmpdir experiments = get_experiments(experiment_folder,load=True,warning=False) if experiment_tags != None: experiments = [e for e in experiments if e[0]["exp_id"] in experiment_tags] # We need a dictionary to look up experiments by task ids unique_tasks = numpy.unique([e[0]["cognitive_atlas_task_id"] for e in experiments]).tolist() experiment_lookup = dict() for u in unique_tasks: matching_tasks = numpy.unique([e[0]["exp_id"] for e in experiments if e[0]["cognitive_atlas_task_id"]==u]) experiment_lookup[u] = matching_tasks.tolist() triples = concept_node_triples(image_dict=experiment_lookup,save_to_file=False,lookup_key_type="task") # Experiments not in the tree get added to parent node 1 undefined_experiments = [x[0]["exp_id"] for x in experiments if x[0]["exp_id"] not in triples.name.tolist()] undefined_experiments.sort() last_defined_node = [x for x in triples.id.tolist() if re.search("node_",x)] last_defined_node.sort() last_defined_node = int(last_defined_node[-1].replace("node_","")) idx = triples.index.tolist()[-1]+1 for i in range(idx,idx+len(undefined_experiments)): undefined_experiment = undefined_experiments.pop(0) triples.loc[i] = ["node_%s" %(last_defined_node+1),1,undefined_experiment] last_defined_node+=1 # We want to add meta data for the experiments meta_data = dict() for experiment in experiments: node_ids = triples.id[triples.name==experiment[0]["exp_id"]].tolist() for node_id in node_ids: meta_data[node_id] = experiment[0] # Generate a data structure with task/concept hierarchy, prune_tree default is True if get_html == True: tree = make_tree_from_triples(triples,output_html=True,meta_data=meta_data) else: tree = make_tree_from_triples(triples,output_html=False) return tree
def get_experiment_selection(repo_urls, tmpdir=None, remove_tmp=True): '''get_experiment_selection will return a list of valid experiments from a github repo :param repo_urls: a list of github repo urls :param tmpdir: provide a custom temporary directory, will be generated if not provided :param remove_tmp: delete the temporary directory at finish (default True) ''' tmpdir = download_repos(repo_urls, tmpdir=tmpdir) # Get all experiments in subfolders subfolders = glob("%s/*" % tmpdir) experiments = [] for subfolder in subfolders: new_experiments = get_experiments(subfolder, load=True, warning=False) experiments = experiments + new_experiments experiments = [x[0] for x in experiments] if remove_tmp == True: shutil.rmtree(tmpdir) return experiments
def setup(self): ''' obtain database and filesystem preferences from defaults, and compare with selection in container. ''' self.selection = EXPFACTORY_EXPERIMENTS self.data_base = EXPFACTORY_DATA self.study_id = EXPFACTORY_SUBID self.base = EXPFACTORY_BASE self.randomize = EXPFACTORY_RANDOMIZE available = get_experiments("%s" % self.base) self.experiments = get_selection(available, self.selection) self.logger.debug(self.experiments) self.lookup = make_lookup(self.experiments) final = "\n".join(list(self.lookup.keys())) bot.log("User has selected: %s" % self.selection) bot.log("Experiments Available: %s" % "\n".join(available)) bot.log("Randomize: %s" % self.randomize) bot.log("Final Set \n%s" % final)
def validate_circle_yml(experiment_repo,repo_type="experiments"): '''validate_circle_yml will make sure that all experiment folders in a repo are tested with circle_ci_test in the circle.yml :param experiment_repo: the experiment repo folder that contains experiment subdirectories with config.json files ''' if "CIRCLE_BRANCH" in os.environ.keys(): circle_yml_file = "%s/circle.yml" %experiment_repo assert_equal(os.path.exists(circle_yml_file),True) circle_yml_file = open(circle_yml_file,"r") circle_yml = "".join([x.strip("\n").replace(" ","").replace("'","").replace('"',"") for x in circle_yml_file.readlines()]) circle_yml = circle_yml.replace("(","").replace(")","") experiments = get_experiments(experiment_repo,load=True,warning=False,repo_type=repo_type) tags = [x[0]["exp_id"] for x in experiments] for tag in tags: print "TESTING if %s defined for circle ci testing in circle.yml..." %tag if repo_type == "experiments": assert_equal(re.search("circle_ci_test%s" %tag,circle_yml)!=None,True) elif repo_type == "surveys": assert_equal(re.search("circle_ci_survey%s" %tag,circle_yml)!=None,True) else: print "Not in a continuous integration (CircleCI) environment, skipping test." print "All experiments found in circle.yml for testing!"
def get_experiment_selection(repo_type="experiments"): tmpdir = custom_battery_download(repos=[repo_type]) experiments = get_experiments("%s/%s" % (tmpdir, repo_type), load=True, warning=False, repo_type=repo_type) experiments = [x[0] for x in experiments] shutil.rmtree(tmpdir) return experiments
def get_experiment_selection(repo_type="experiments"): tmpdir = custom_battery_download(repos=[repo_type]) experiments = get_experiments("%s/%s" %(tmpdir,repo_type),load=True,warning=False,repo_type=repo_type) experiments = [x[0] for x in experiments] shutil.rmtree(tmpdir) return experiments
def get_experiment_selection(): tmpdir = custom_battery_download(repos=["experiments"]) experiments = get_experiments("%s/experiments" %tmpdir,load=True,warning=False) experiments = [x[0] for x in experiments] shutil.rmtree(tmpdir) return experiments
def generate_base(battery_dest, tasks=None, experiment_repo=None, survey_repo=None, game_repo=None, battery_repo=None, warning=True): '''generate_base returns a folder with downloaded experiments, surveys, and battery, either specified by the user or a temporary directory, to be used by generate_local and generate (for psiturk) :param battery_dest: [required] is the output folder for your battery. This folder MUST NOT EXIST. :param battery_repo: location of psiturk-battery repo to use as a template. If not specified, will be downloaded to a temporary directory :param experiment_repo: location of a expfactory-experiments repo to check for valid experiments. If not specified, will be downloaded to a temporary directory :param survey_repo: location of a expfactory-surveys repo to check for valid surveys. If not specified, will be downloaded to a temporary directory :param tasks: a list of experiments and surveys, meaning the "exp_id" variable in the config.json, to include. This variable also conincides with the tasks folder name. :param warning: show warnings when validating experiments (default True) ''' repos = [] tmpdir = None if battery_repo == None: tmpdir = custom_battery_download(repos=['battery']) battery_repo = "%s/battery" % (tmpdir) if experiment_repo == None and survey_repo == None and game_repo == None: tmpdir = custom_battery_download(tmpdir, ['experiments', 'surveys', 'games']) experiment_repo = "%s/experiments" % (tmpdir) survey_repo = "%s/surveys" % (tmpdir) game_repo = "%s/games" % (tmpdir) # Copy battery skeleton to destination copy_directory(battery_repo, battery_dest) valid_experiments = [] valid_surveys = [] valid_games = [] if experiment_repo is not None: valid_experiments = get_experiments(experiment_repo, warning=warning) if survey_repo is not None: valid_surveys = get_experiments(survey_repo, warning=warning, repo_type="surveys") if game_repo is not None: valid_games = get_experiments(game_repo, warning=warning, repo_type="games") # If the user wants to select a subset if tasks != None: valid_experiments = [ x for x in valid_experiments if os.path.basename(x) in [os.path.basename(e) for e in tasks] ] valid_surveys = [ x for x in valid_surveys if os.path.basename(x) in [os.path.basename(e) for e in tasks] ] valid_games = [ x for x in valid_games if os.path.basename(x) in [os.path.basename(e) for e in tasks] ] base = { "battery_repo": battery_repo, "experiment_repo": experiment_repo, "survey_repo": survey_repo, "game_repo": game_repo, "experiments": valid_experiments, "surveys": valid_surveys, "games": valid_games } return base
def generate_experiment_web(output_dir,experiment_folder=None,survey_folder=None,games_folder=None, make_table=True,make_index=True,make_experiments=True,make_data=True, make_surveys=True,make_games=True): '''get_experiment_table Generate a table with links to preview all experiments :param experiment_folder: folder with experiments inside :param survey_folder: folder with surveys inside :param games_folder: folder with games inside :param output_dir: output folder for experiment and table html, and battery files :param make_table: generate table.html :param make_index: generate d3 visualization index :param make_experiments: generate experiment preview files (linked from table and index) :param make_data: generate json/tsv data to download :param make_surveys: generate static files for surveys repos :param make_games: generate static files for games repos ''' repos=["experiments","battery"] if make_surveys == True: repos.append("surveys") if make_games == True: repos.append("games") tmpdir = custom_battery_download(repos=repos) if not os.path.exists(output_dir): os.mkdir(output_dir) if experiment_folder == None: experiment_folder = "%s/experiments" %tmpdir experiments = get_experiments(experiment_folder,load=True, warning=False) experiment_tags = [x[0]["exp_id"] for x in experiments] battery_repo = "%s/battery" %(tmpdir) if survey_folder == None: survey_folder = "%s/surveys" %tmpdir if games_folder == None: games_folder = "%s/games" %tmpdir # If the user wants surveys and/or games, add them on to tasks tasks = experiments if make_surveys == True: surveys = get_experiments(survey_folder,load=True,warning=False,repo_type="surveys") survey_tags = [x[0]["exp_id"] for x in surveys] tasks = experiments + surveys if make_games == True: games = get_experiments(games_folder,load=True,warning=False,repo_type="games") games_tags = [x[0]["exp_id"] for x in games] tasks = tasks + games # Fields to keep for the table fields = ['preview','exp_id','template', 'contributors','time', 'cognitive_atlas_task_id'] valid = pandas.DataFrame(columns=fields) # Make a table of experiment information for experiment in tasks: for field in experiment[0].keys(): if field in fields: values = experiment[0][field] # Join lists with a comma if field == "reference": if values != '': values = '<a href="%s" target="_blank">%s</a>' %(values,values) if isinstance(values,list): values = ",".join(values) valid.loc[experiment[0]["exp_id"],field] = values # Add a preview link valid.loc[experiment[0]["exp_id"],"preview"] = '<a href="%s.html" target="_blank">DEMO</a>' %(experiment[0]["exp_id"]) # If the user wants to create the index page if make_index == True: output_index = os.path.abspath("%s/index.html" %output_dir) # For each experiment, we will prepare an interactive node for the site nodes = [] for experiment in tasks: nodes.append('{"cluster": 1, "radius": "10", "color": colors[%s], "exp_id": "%s" }' %(choice([0,1,2]),experiment[0]["exp_id"])) # Generate index page index_template = get_template("%s/templates/expfactory_index.html" %get_installdir()) index_template = index_template.replace("[SUB_NODES_SUB]",",".join(nodes)) index_template = index_template.replace("[SUB_TOTAL_SUB]",str(len(nodes))) filey = open(output_index,"wb") filey.writelines(index_template) filey.close() # Update entire static directory old_dirs = ["templates","static"] for folder in old_dirs: copy_to = os.path.abspath("%s/%s" %(output_dir,folder)) copy_from = "%s/battery/%s" %(tmpdir,folder) if os.path.exists(copy_to): shutil.rmtree(copy_to) copy_directory(copy_from,copy_to) # Copy the favicon to web base shutil.copyfile("%s/battery/static/favicon.ico" %tmpdir,"%s/favicon.ico" %output_dir) # Clear old experiments experiment_dir = os.path.abspath("%s/static/experiments/" %output_dir) if os.path.exists(experiment_dir): shutil.rmtree(experiment_dir) # Clear old surveys, copy updated valid surveys into survey directory if make_surveys == True: survey_dir = os.path.abspath("%s/static/surveys/" %output_dir) if os.path.exists(survey_dir): shutil.rmtree(survey_dir) valid_surveys = ["%s/%s" %(survey_folder,x[0]["exp_id"]) for x in surveys] move_experiments(valid_surveys,battery_dest=output_dir,repo_type="surveys") # Clear old surveys, copy updated valid surveys into survey directory if make_games == True: games_dir = os.path.abspath("%s/static/games/" %output_dir) if os.path.exists(games_dir): shutil.rmtree(games_dir) valid_games = ["%s/%s" %(games_folder,x[0]["exp_id"]) for x in games] move_experiments(valid_games,battery_dest=output_dir,repo_type="games") # Copy updated valid experiments into our experiment directory valid_experiments = ["%s/%s" %(experiment_folder,x[0]["exp_id"]) for x in experiments] move_experiments(valid_experiments,battery_dest=output_dir) # If the user wants to make a table if make_table == True: table_template = get_template("%s/templates/table.html" %get_installdir()) output_table = os.path.abspath("%s/table.html" %output_dir) # First prepare rendered table table = '<table id="fresh-table" class="table">\n<thead>\n' for field in fields: table = '%s<th data-field="%s" data-sortable="true">%s</th>' %(table,field,field) table = '%s\n</thead>\n<tbody>\n' %(table) for row in valid.iterrows(): table = "%s<tr>\n" %(table) for field in row[1]: table = "%s<td>%s</td>\n" %(table,field) table = "%s</tr>\n" %(table) table = "%s</tbody></table>\n" %(table) # Write the new table table_template = table_template.replace("[[SUB_TABLE_SUB]]",table) filey = open("%s/table.html" %output_dir,"wb") filey.writelines(table_template) filey.close() if make_experiments == True: experiments_template = get_template("%s/templates/experiments_categories.html" %get_installdir()) output_exp = os.path.abspath("%s/experiments.html" %output_dir) if "CIRCLE_BRANCH" in os.environ.keys(): experiment_page = table_template else: experiment_page = get_cognitiveatlas_hierarchy(experiment_tags=experiment_tags,get_html=True) # Write the new table filey = open(output_exp,"wb") filey.writelines(experiment_page) filey.close() # For each experiment, we will generate a demo page for experiment in experiments: demo_page = os.path.abspath("%s/%s.html" %(output_dir,experiment[0]["exp_id"])) exp_template = get_experiment_html(experiment,"%s/%s" %(experiment_folder,experiment[0]["exp_id"])) filey = open(demo_page,"wb") filey.writelines(exp_template) filey.close() # if the user wants to make surveys if make_surveys == True: for experiment in surveys: demo_page = os.path.abspath("%s/%s.html" %(output_dir,experiment[0]["exp_id"])) exp_template = get_experiment_html(experiment,"%s/%s" %(survey_folder,experiment[0]["exp_id"])) filey = open(demo_page,"wb") filey.writelines(exp_template) filey.close() # if the user wants to make surveys if make_games == True: for experiment in games: demo_page = os.path.abspath("%s/%s.html" %(output_dir,experiment[0]["exp_id"])) exp_template = get_experiment_html(experiment,"%s/%s" %(games_folder,experiment[0]["exp_id"])) filey = open(demo_page,"wb") filey.writelines(exp_template) filey.close() # If the user wants to make data if make_data == True: data_folder = os.path.abspath("%s/data" %output_dir) if not os.path.exists(data_folder): os.mkdir(data_folder) save_pretty_json("%s/expfactory-experiments.json" %(data_folder),json.loads(valid.to_json(orient="records"))) valid.to_csv("%s/expfactory-experiments.tsv" %(data_folder),sep="\t",index=None) valid.to_pickle("%s/expfactory-experiments.pkl" %(data_folder))