def test_find_experiments(self):
# Test loading experiment folder path
experiments = get_experiments(self.battery_folder)
assert_equal(len(experiments), 1)
self.assertTrue(isinstance(experiments[0], str))
# Test loading experiment dictionary
config = get_experiments(self.battery_folder, load=True)
self.assertTrue(isinstance(config[0][0], dict))
loaded_experiment = load_experiment(self.experiment)
self.assertTrue(isinstance(loaded_experiment[0], dict))
def test_find_experiments(self):
# Test loading experiment folder path
experiments = get_experiments(self.battery_folder)
assert_equal(len(experiments),1)
self.assertTrue(isinstance(experiments[0],str))
# Test loading experiment dictionary
config = get_experiments(self.battery_folder,load=True)
self.assertTrue(isinstance(config[0][0],dict))
loaded_experiment = load_experiment(self.experiment)
self.assertTrue(isinstance(loaded_experiment[0],dict))
def generate_base(battery_dest,tasks=None,experiment_repo=None,survey_repo=None,game_repo=None,
add_experiments=True,add_surveys=True,add_games=True,battery_repo=None,warning=True):
'''generate_base returns a folder with downloaded experiments, surveys, and battery, either specified by the user or a temporary directory, to be used by generate_local and generate (for psiturk)
:param battery_dest: [required] is the output folder for your battery. This folder MUST NOT EXIST.
:param battery_repo: location of psiturk-battery repo to use as a template. If not specified, will be downloaded to a temporary directory
:param experiment_repo: location of a expfactory-experiments repo to check for valid experiments. If not specified, will be downloaded to a temporary directory
:param survey_repo: location of a expfactory-surveys repo to check for valid surveys. If not specified, will be downloaded to a temporary directory
:param tasks: a list of experiments and surveys, meaning the "exp_id" variable in the config.json, to include. This variable also conincides with the tasks folder name.
:param warning: show warnings when validating experiments (default True)
'''
if experiment_repo == None or battery_repo == None or survey_repo == None or game_repo == None:
tmpdir = custom_battery_download()
if experiment_repo == None:
experiment_repo = "%s/experiments" %(tmpdir)
if battery_repo == None:
battery_repo = "%s/battery" %(tmpdir)
if survey_repo == None:
survey_repo = "%s/surveys" %(tmpdir)
if game_repo == None:
game_repo = "%s/games" %(tmpdir)
# Copy battery skeleton to destination
copy_directory(battery_repo,battery_dest)
valid_experiments = []
valid_surveys = []
valid_games = []
if add_experiments == True:
valid_experiments = get_experiments(experiment_repo,warning=warning)
if add_surveys == True:
valid_surveys = get_experiments(survey_repo,warning=warning,repo_type="surveys")
if add_games == True:
valid_games = get_experiments(game_repo,warning=warning,repo_type="games")
# If the user wants to select a subset
if tasks != None:
valid_experiments = [x for x in valid_experiments if os.path.basename(x) in [os.path.basename(e) for e in tasks]]
valid_surveys = [x for x in valid_surveys if os.path.basename(x) in [os.path.basename(e) for e in tasks]]
valid_games = [x for x in valid_games if os.path.basename(x) in [os.path.basename(e) for e in tasks]]
base = {"battery_repo":battery_repo,
"experiment_repo":experiment_repo,
"survey_repo":survey_repo,
"game_repo":game_repo,
"experiments":valid_experiments,
"surveys":valid_surveys,
"games":valid_games}
return base
def get_surveys(survey_repo=None,load=False,warning=True,repo_type="surveys"):
'''get_surveys is a wrapper for "get_experiments" - the functionality is the same, but provided for users: return loaded json for all valid survyes from an surveys folder
:param survey_repo: full path to the surveys repo
:param load: if True, returns a list of loaded config.json objects. If False (default) returns the paths to the experiments
:param repo_type: tells the user what kind of task is being parsed, default is "experiments," but can also be "surveys" when called by get_surveys
'''
return get_experiments(experiment_repo=survey_repo,load=load,warning=warning,repo_type=repo_type)
def setup(self):
''' obtain database and filesystem preferences from defaults,
and compare with selection in container.
'''
self.selection = EXPFACTORY_EXPERIMENTS
self.ordered = len(EXPFACTORY_EXPERIMENTS) > 0
self.data_base = EXPFACTORY_DATA
self.study_id = EXPFACTORY_SUBID
self.base = EXPFACTORY_BASE
self.randomize = EXPFACTORY_RANDOMIZE
self.headless = EXPFACTORY_HEADLESS
# Generate variables, if they exist
self.vars = generate_runtime_vars() or None
available = get_experiments("%s" % self.base)
self.experiments = get_selection(available, self.selection)
self.logger.debug(self.experiments)
self.lookup = make_lookup(self.experiments)
final = "\n".join(list(self.lookup.keys()))
bot.log("Headless mode: %s" % self.headless)
bot.log("User has selected: %s" % self.selection)
bot.log("Experiments Available: %s" %"\n".join(available))
bot.log("Randomize: %s" % self.randomize)
bot.log("Final Set \n%s" % final)
def setup(self):
""" obtain database and filesystem preferences from defaults,
and compare with selection in container.
"""
self.selection = EXPFACTORY_EXPERIMENTS
self.ordered = len(EXPFACTORY_EXPERIMENTS) > 0
self.data_base = EXPFACTORY_DATA
self.study_id = EXPFACTORY_SUBID
self.base = EXPFACTORY_BASE
self.randomize = EXPFACTORY_RANDOMIZE
self.headless = EXPFACTORY_HEADLESS
# Generate variables, if they exist
self.vars = generate_runtime_vars() or None
available = get_experiments("%s" % self.base)
self.experiments = get_selection(available, self.selection)
self.logger.debug(self.experiments)
self.lookup = make_lookup(self.experiments)
final = "\n".join(list(self.lookup.keys()))
bot.log("Headless mode: %s" % self.headless)
bot.log("User has selected: %s" % self.selection)
bot.log("Experiments Available: %s" % "\n".join(available))
bot.log("Randomize: %s" % self.randomize)
bot.log("Final Set \n%s" % final)
def validate_circle_yml(experiment_repo, repo_type="experiments"):
'''validate_circle_yml will make sure that all experiment folders in a repo are tested with circle_ci_test in the circle.yml
:param experiment_repo: the experiment repo folder that contains experiment subdirectories with config.json files
'''
if "CIRCLE_BRANCH" in os.environ.keys():
circle_yml_file = "%s/circle.yml" % experiment_repo
assert_equal(os.path.exists(circle_yml_file), True)
circle_yml_file = open(circle_yml_file, "r")
circle_yml = "".join([
x.strip("\n").replace(" ", "").replace("'", "").replace('"', "")
for x in circle_yml_file.readlines()
])
circle_yml = circle_yml.replace("(", "").replace(")", "")
experiments = get_experiments(experiment_repo,
load=True,
warning=False,
repo_type=repo_type)
tags = [x[0]["exp_id"] for x in experiments]
for tag in tags:
print "TESTING if %s defined for circle ci testing in circle.yml..." % tag
if repo_type == "experiments":
assert_equal(
re.search("circle_ci_test%s" % tag, circle_yml) != None,
True)
elif repo_type == "surveys":
assert_equal(
re.search("circle_ci_survey%s" % tag, circle_yml) != None,
True)
else:
print "Not in a continuous integration (CircleCI) environment, skipping test."
print "All experiments found in circle.yml for testing!"
def experiment_robot_web(web_folder,experiment_tags=None,port=None,pause_time=100):
'''experiment_robot_web
Robot to automatically run and test experiments, to work with an experiment web folder (meaning produced with views.get_experiment_web. This folder has the standard battery structure with experiment pre-generated as html files. A separate function will/should eventually be made for single experiment preview.
:param experiment_tags: experimentweb generated folder with generate_experiment_web
:param experiment_tags: list of experiment folders to test
:param pause_time: time to wait between tasks, in addition to time specified in jspsych
:param port: port. Randomly selected if None is selected
'''
experimentweb_base = os.path.abspath(web_folder)
if port == None:
port = choice(range(8000,9999),1)[0]
Handler = ExpfactoryServer
httpd = SocketServer.TCPServer(("", port), Handler)
server = Thread(target=httpd.serve_forever)
server.setDaemon(True)
server.start()
# Set up a web browser
os.chdir(experimentweb_base)
browser = get_browser()
browser.implicitly_wait(3) # if error, will wait 3 seconds and retry
browser.set_page_load_timeout(10)
# Find experiments
experiments = get_experiments("%s/static/experiments" %experimentweb_base,load=True,warning=False)
if experiment_tags != None:
experiments = [e for e in experiments if e[0]["exp_id"] in experiment_tags]
print "Found %s experiments to test." %(len(experiments))
for experiment in experiments:
print "STARTING TEST OF EXPERIMENT %s" %(experiment[0]["exp_id"])
get_page(browser,"http://localhost:%s/%s.html" %(port,experiment[0]["exp_id"]))
sleep(3)
count=0
wait_time=0
while True:
print "Testing command %s of %s" %(count,experiment[0]["exp_id"])
try:
wait_time,finished = test_block(browser,experiment,pause_time,wait_time)
if finished == True:
break
count+=1
except UnexpectedAlertPresentException:
print "Found alert: closing."
try:
alert = browser.switch_to_alert()
alert.accept()
except:
pass
print "FINISHING TEST OF EXPERIMENT %s" %(experiment[0]["exp_id"])
# Stop the server
httpd.server_close()
def install_experiments(experiment_tags=None, repo_type="experiments"):
# We will return list of experiments that did not install successfully
errored_experiments = []
tmpdir = custom_battery_download(repos=[repo_type, "battery"])
# The git commit is saved with the experiment as the "version"
repo = Repo("%s/%s" % (tmpdir, repo_type))
commit = repo.commit("master").__str__()
experiments = get_experiments("%s/%s" % (tmpdir, repo_type), load=True, warning=False)
if experiment_tags != None:
experiments = [e for e in experiments if e[0]["exp_id"] in experiment_tags]
for experiment in experiments:
try:
performance_variable = None
rejection_variable = None
if "experiment_variables" in experiment[0]:
if isinstance(experiment[0]["experiment_variables"], list):
for var in experiment[0]["experiment_variables"]:
if var["type"].lower().strip() == "bonus":
performance_variable = parse_experiment_variable(var)
elif var["type"].lower().strip() == "credit":
rejection_variable = parse_experiment_variable(var)
else:
parse_experiment_variable(var) # adds to database
if isinstance(experiment[0]["reference"], list):
reference = experiment[0]["reference"][0]
else:
reference = experiment[0]["reference"]
cognitive_atlas_task = get_cognitiveatlas_task(experiment[0]["cognitive_atlas_task_id"])
new_experiment, _ = ExperimentTemplate.objects.update_or_create(
exp_id=experiment[0]["exp_id"],
defaults={
"name": experiment[0]["name"],
"cognitive_atlas_task": cognitive_atlas_task,
"publish": bool(experiment[0]["publish"]),
"time": experiment[0]["time"],
"reference": reference,
"version": commit,
"template": experiment[0]["template"],
"performance_variable": performance_variable,
"rejection_variable": rejection_variable,
},
)
new_experiment.save()
experiment_folder = "%s/%s/%s" % (tmpdir, repo_type, experiment[0]["exp_id"])
output_folder = "%s/%s/%s" % (media_dir, repo_type, experiment[0]["exp_id"])
if os.path.exists(output_folder):
shutil.rmtree(output_folder)
copy_directory(experiment_folder, output_folder)
except:
errored_experiments.append(experiment[0]["exp_id"])
shutil.rmtree(tmpdir)
return errored_experiments
def __init__(self, *args, **kwargs):
super(EFServer, self).__init__(*args, **kwargs)
# download repo on start of application
self.tmpdir = tempfile.mkdtemp()
custom_battery_download(tmpdir=self.tmpdir)
self.experiments = get_experiments("%s/experiments" %self.tmpdir,load=True,warning=False)
self.experiment_lookup = make_lookup(self.experiments,"exp_id")
def test_generate(self):
# Test loading experiment folder path
generate(self.battery)
self.assertTrue(os.path.exists(self.battery))
# Test loading experiments
experiments = get_experiments("%s/static/experiments" %self.battery,load=True)
self.assertTrue(len(experiments)>0)
def survey_robot_web(web_folder, survey_tags=None, port=None, pause_time=100):
'''survey_robot_web
Robot to automatically click through surveys, to work with an experiment web folder (meaning produced with views.get_experiment_web. This folder has the standard battery structure with experiment pre-generated as html files.
:param web_folder: experimentweb generated folder with generate_experiment_web
:param survey_tags: list of experiment folders to test
:param pause_time: time to wait between tasks, in addition to time specified in jspsych
:param port: port. Randomly selected if None is selected
'''
web_base = os.path.abspath(web_folder)
httpd, port = get_web_server(port=port)
# Set up a web browser
os.chdir(web_base)
browser = get_browser()
browser.implicitly_wait(3) # if error, will wait 3 seconds and retry
browser.set_page_load_timeout(10)
# Find surveys
surveys = get_experiments("%s/static/surveys" % web_base,
load=True,
warning=False,
repo_type="surveys")
if survey_tags != None:
surveys = [s for s in surveys if s[0]["exp_id"] in survey_tags]
print "Found %s surveys to test." % (len(surveys))
for survey in surveys:
print "STARTING TEST OF SURVEY %s" % (survey[0]["exp_id"])
get_page(browser,
"http://localhost:%s/%s.html" % (port, survey[0]["exp_id"]))
sleep(3)
count = 1
while True:
print "Testing page %s of %s" % (count, survey[0]["exp_id"])
try:
finished = advance_survey(browser, pause_time)
if finished == True:
break
count += 1
except UnexpectedAlertPresentException:
print "Found alert: closing."
try:
alert = browser.switch_to_alert()
alert.accept()
except:
pass
print "FINISHING TEST OF SURVEY %s" % (survey[0]["exp_id"])
# Stop the server
httpd.server_close()
def get_experiment_selection(repo_type="experiments"):
# tmpdir = custom_battery_download(repos=[repo_type])
tmpdir = os.path.join(EXP_REPO, 'expfactory-{}'.format(repo_type))
experiments = get_experiments(tmpdir,
load=True,
warning=False,
repo_type=repo_type)
experiments = [x[0] for x in experiments]
# shutil.rmtree(tmpdir)
return experiments
def test_generate(self):
# Test loading experiment folder path
generate(self.battery)
self.assertTrue(os.path.exists(self.battery))
# Test loading experiments
experiments = get_experiments("%s/static/experiments" % self.battery,
load=True)
self.assertTrue(len(experiments) > 0)
def install_experiments(experiment_tags=None,repo_type="experiments"):
# We will return list of experiments that did not install successfully
errored_experiments = []
tmpdir = custom_battery_download(repos=[repo_type,"battery"])
# The git commit is saved with the experiment as the "version"
repo = Repo("%s/%s" %(tmpdir,repo_type))
commit = repo.commit('master').__str__()
experiments = get_experiments("%s/%s" %(tmpdir,repo_type),load=True,warning=False)
if experiment_tags != None:
experiments = [e for e in experiments if e[0]["exp_id"] in experiment_tags]
for experiment in experiments:
try:
performance_variable = None
rejection_variable = None
if "experiment_variables" in experiment[0]:
if isinstance(experiment[0]["experiment_variables"],list):
for var in experiment[0]["experiment_variables"]:
if var["type"].lower().strip() == "bonus":
performance_variable = parse_experiment_variable(var)
elif var["type"].lower().strip() == "credit":
rejection_variable = parse_experiment_variable(var)
else:
parse_experiment_variable(var) # adds to database
if isinstance(experiment[0]["reference"],list):
reference = experiment[0]["reference"][0]
else:
reference = experiment[0]["reference"]
cognitive_atlas_task = get_cognitiveatlas_task(experiment[0]["cognitive_atlas_task_id"])
new_experiment,_ = ExperimentTemplate.objects.update_or_create(exp_id=experiment[0]["exp_id"],
defaults={"name":experiment[0]["name"],
"cognitive_atlas_task":cognitive_atlas_task,
"publish":bool(experiment[0]["publish"]),
"time":experiment[0]["time"],
"reference":reference,
"version":commit,
"template":experiment[0]["template"],
"performance_variable":performance_variable,
"rejection_variable":rejection_variable})
new_experiment.save()
experiment_folder = "%s/%s/%s" %(tmpdir,repo_type,experiment[0]["exp_id"])
output_folder = "%s/%s/%s" %(media_dir,repo_type,experiment[0]["exp_id"])
if os.path.exists(output_folder):
shutil.rmtree(output_folder)
copy_directory(experiment_folder,output_folder)
except:
errored_experiments.append(experiment[0]["exp_id"])
shutil.rmtree(tmpdir)
return errored_experiments
def experiment_robot_web(web_folder,experiment_tags=None,port=None,pause_time=100):
'''experiment_robot_web
Robot to automatically run and test experiments, to work with an experiment web folder (meaning produced with views.get_experiment_web. This folder has the standard battery structure with experiment pre-generated as html files.
:param web_folder: experimentweb generated folder with generate_experiment_web
:param experiment_tags: list of experiment folders to test
:param pause_time: time to wait between tasks, in addition to time specified in jspsych
:param port: port. Randomly selected if None is selected
'''
experimentweb_base = os.path.abspath(web_folder)
httpd,port = get_web_server(port=port)
# Set up a web browser
os.chdir(experimentweb_base)
browser = get_browser()
browser.implicitly_wait(3) # if error, will wait 3 seconds and retry
browser.set_page_load_timeout(10)
# Find experiments
experiments = get_experiments("%s/static/experiments" %experimentweb_base,load=True,warning=False)
if experiment_tags != None:
experiments = [e for e in experiments if e[0]["exp_id"] in experiment_tags]
print "Found %s experiments to test." %(len(experiments))
for experiment in experiments:
print "STARTING TEST OF EXPERIMENT %s" %(experiment[0]["exp_id"])
get_page(browser,"http://localhost:%s/%s.html" %(port,experiment[0]["exp_id"]))
sleep(3)
count=0
wait_time=0
while True:
print "Testing command %s of %s" %(count,experiment[0]["exp_id"])
try:
wait_time,finished = test_block(browser,experiment,pause_time,wait_time)
if finished == True:
break
count+=1
except UnexpectedAlertPresentException:
print "Found alert: closing."
try:
alert = browser.switch_to_alert()
alert.accept()
except:
pass
print "FINISHING TEST OF EXPERIMENT %s" %(experiment[0]["exp_id"])
# Stop the server
httpd.server_close()
def survey_robot_web(web_folder,survey_tags=None,port=None,pause_time=100):
'''survey_robot_web
Robot to automatically click through surveys, to work with an experiment web folder (meaning produced with views.get_experiment_web. This folder has the standard battery structure with experiment pre-generated as html files.
:param web_folder: experimentweb generated folder with generate_experiment_web
:param survey_tags: list of experiment folders to test
:param pause_time: time to wait between tasks, in addition to time specified in jspsych
:param port: port. Randomly selected if None is selected
'''
web_base = os.path.abspath(web_folder)
httpd,port = get_web_server(port=port)
# Set up a web browser
os.chdir(web_base)
browser = get_browser()
browser.implicitly_wait(3) # if error, will wait 3 seconds and retry
browser.set_page_load_timeout(10)
# Find surveys
surveys = get_experiments("%s/static/surveys" %web_base,load=True,warning=False,repo_type="surveys")
if survey_tags != None:
surveys = [s for s in surveys if s[0]["exp_id"] in survey_tags]
print "Found %s surveys to test." %(len(surveys))
for survey in surveys:
print "STARTING TEST OF SURVEY %s" %(survey[0]["exp_id"])
get_page(browser,"http://localhost:%s/%s.html" %(port,survey[0]["exp_id"]))
sleep(3)
count=1
while True:
print "Testing page %s of %s" %(count,survey[0]["exp_id"])
try:
finished = advance_survey(browser,pause_time)
if finished == True:
break
count+=1
except UnexpectedAlertPresentException:
print "Found alert: closing."
try:
alert = browser.switch_to_alert()
alert.accept()
except:
pass
print "FINISHING TEST OF SURVEY %s" %(survey[0]["exp_id"])
# Stop the server
httpd.server_close()
def get_surveys(survey_repo=None,
load=False,
warning=True,
repo_type="surveys"):
'''get_surveys is a wrapper for "get_experiments" - the functionality is the same, but provided for users: return loaded json for all valid survyes from an surveys folder
:param survey_repo: full path to the surveys repo
:param load: if True, returns a list of loaded config.json objects. If False (default) returns the paths to the experiments
:param repo_type: tells the user what kind of task is being parsed, default is "experiments," but can also be "surveys" when called by get_surveys
'''
return get_experiments(experiment_repo=survey_repo,
load=load,
warning=warning,
repo_type=repo_type)
def get_cognitiveatlas_hierarchy(experiment_tags=None,get_html=False):
'''get_cognitiveatlas_hierarchy
return
:param experiment_tags: a list of experiment exp_id tags to include. If None provided, all valid experiments will be used.
:param get_html: if True, returns rendered HTML template with hierarchy. False returns json data structure.
'''
from cognitiveatlas.datastructure import concept_node_triples
from expfactory.graph import make_tree_from_triples
tmpdir = custom_battery_download()
experiment_folder = "%s/experiments" %tmpdir
experiments = get_experiments(experiment_folder,load=True,warning=False)
if experiment_tags != None:
experiments = [e for e in experiments if e[0]["exp_id"] in experiment_tags]
# We need a dictionary to look up experiments by task ids
unique_tasks = numpy.unique([e[0]["cognitive_atlas_task_id"] for e in experiments]).tolist()
experiment_lookup = dict()
for u in unique_tasks:
matching_tasks = numpy.unique([e[0]["exp_id"] for e in experiments if e[0]["cognitive_atlas_task_id"]==u])
experiment_lookup[u] = matching_tasks.tolist()
triples = concept_node_triples(image_dict=experiment_lookup,save_to_file=False,lookup_key_type="task")
# Experiments not in the tree get added to parent node 1
undefined_experiments = [x[0]["exp_id"] for x in experiments if x[0]["exp_id"] not in triples.name.tolist()]
undefined_experiments.sort()
last_defined_node = [x for x in triples.id.tolist() if re.search("node_",x)]
last_defined_node.sort()
last_defined_node = int(last_defined_node[-1].replace("node_",""))
idx = triples.index.tolist()[-1]+1
for i in range(idx,idx+len(undefined_experiments)):
undefined_experiment = undefined_experiments.pop(0)
triples.loc[i] = ["node_%s" %(last_defined_node+1),1,undefined_experiment]
last_defined_node+=1
# We want to add meta data for the experiments
meta_data = dict()
for experiment in experiments:
node_ids = triples.id[triples.name==experiment[0]["exp_id"]].tolist()
for node_id in node_ids:
meta_data[node_id] = experiment[0]
# Generate a data structure with task/concept hierarchy, prune_tree default is True
if get_html == True:
tree = make_tree_from_triples(triples,output_html=True,meta_data=meta_data)
else:
tree = make_tree_from_triples(triples,output_html=False)
return tree
def get_experiment_selection(repo_urls, tmpdir=None, remove_tmp=True):
'''get_experiment_selection will return a list of valid experiments from a github repo
:param repo_urls: a list of github repo urls
:param tmpdir: provide a custom temporary directory, will be generated if not provided
:param remove_tmp: delete the temporary directory at finish (default True)
'''
tmpdir = download_repos(repo_urls, tmpdir=tmpdir)
# Get all experiments in subfolders
subfolders = glob("%s/*" % tmpdir)
experiments = []
for subfolder in subfolders:
new_experiments = get_experiments(subfolder, load=True, warning=False)
experiments = experiments + new_experiments
experiments = [x[0] for x in experiments]
if remove_tmp == True:
shutil.rmtree(tmpdir)
return experiments
def setup(self):
''' obtain database and filesystem preferences from defaults,
and compare with selection in container.
'''
self.selection = EXPFACTORY_EXPERIMENTS
self.data_base = EXPFACTORY_DATA
self.study_id = EXPFACTORY_SUBID
self.base = EXPFACTORY_BASE
self.randomize = EXPFACTORY_RANDOMIZE
available = get_experiments("%s" % self.base)
self.experiments = get_selection(available, self.selection)
self.logger.debug(self.experiments)
self.lookup = make_lookup(self.experiments)
final = "\n".join(list(self.lookup.keys()))
bot.log("User has selected: %s" % self.selection)
bot.log("Experiments Available: %s" % "\n".join(available))
bot.log("Randomize: %s" % self.randomize)
bot.log("Final Set \n%s" % final)
def validate_circle_yml(experiment_repo,repo_type="experiments"):
'''validate_circle_yml will make sure that all experiment folders in a repo are tested with circle_ci_test in the circle.yml
:param experiment_repo: the experiment repo folder that contains experiment subdirectories with config.json files
'''
if "CIRCLE_BRANCH" in os.environ.keys():
circle_yml_file = "%s/circle.yml" %experiment_repo
assert_equal(os.path.exists(circle_yml_file),True)
circle_yml_file = open(circle_yml_file,"r")
circle_yml = "".join([x.strip("\n").replace(" ","").replace("'","").replace('"',"") for x in circle_yml_file.readlines()])
circle_yml = circle_yml.replace("(","").replace(")","")
experiments = get_experiments(experiment_repo,load=True,warning=False,repo_type=repo_type)
tags = [x[0]["exp_id"] for x in experiments]
for tag in tags:
print "TESTING if %s defined for circle ci testing in circle.yml..." %tag
if repo_type == "experiments":
assert_equal(re.search("circle_ci_test%s" %tag,circle_yml)!=None,True)
elif repo_type == "surveys":
assert_equal(re.search("circle_ci_survey%s" %tag,circle_yml)!=None,True)
else:
print "Not in a continuous integration (CircleCI) environment, skipping test."
print "All experiments found in circle.yml for testing!"
def get_experiment_selection(repo_type="experiments"):
tmpdir = custom_battery_download(repos=[repo_type])
experiments = get_experiments("%s/%s" % (tmpdir, repo_type), load=True, warning=False, repo_type=repo_type)
experiments = [x[0] for x in experiments]
shutil.rmtree(tmpdir)
return experiments
def get_experiment_selection(repo_type="experiments"):
tmpdir = custom_battery_download(repos=[repo_type])
experiments = get_experiments("%s/%s" %(tmpdir,repo_type),load=True,warning=False,repo_type=repo_type)
experiments = [x[0] for x in experiments]
shutil.rmtree(tmpdir)
return experiments
def get_experiment_selection():
tmpdir = custom_battery_download(repos=["experiments"])
experiments = get_experiments("%s/experiments" %tmpdir,load=True,warning=False)
experiments = [x[0] for x in experiments]
shutil.rmtree(tmpdir)
return experiments
def generate_base(battery_dest,
tasks=None,
experiment_repo=None,
survey_repo=None,
game_repo=None,
battery_repo=None,
warning=True):
'''generate_base returns a folder with downloaded experiments, surveys, and battery, either specified by the user or a temporary directory, to be used by generate_local and generate (for psiturk)
:param battery_dest: [required] is the output folder for your battery. This folder MUST NOT EXIST.
:param battery_repo: location of psiturk-battery repo to use as a template. If not specified, will be downloaded to a temporary directory
:param experiment_repo: location of a expfactory-experiments repo to check for valid experiments. If not specified, will be downloaded to a temporary directory
:param survey_repo: location of a expfactory-surveys repo to check for valid surveys. If not specified, will be downloaded to a temporary directory
:param tasks: a list of experiments and surveys, meaning the "exp_id" variable in the config.json, to include. This variable also conincides with the tasks folder name.
:param warning: show warnings when validating experiments (default True)
'''
repos = []
tmpdir = None
if battery_repo == None:
tmpdir = custom_battery_download(repos=['battery'])
battery_repo = "%s/battery" % (tmpdir)
if experiment_repo == None and survey_repo == None and game_repo == None:
tmpdir = custom_battery_download(tmpdir,
['experiments', 'surveys', 'games'])
experiment_repo = "%s/experiments" % (tmpdir)
survey_repo = "%s/surveys" % (tmpdir)
game_repo = "%s/games" % (tmpdir)
# Copy battery skeleton to destination
copy_directory(battery_repo, battery_dest)
valid_experiments = []
valid_surveys = []
valid_games = []
if experiment_repo is not None:
valid_experiments = get_experiments(experiment_repo, warning=warning)
if survey_repo is not None:
valid_surveys = get_experiments(survey_repo,
warning=warning,
repo_type="surveys")
if game_repo is not None:
valid_games = get_experiments(game_repo,
warning=warning,
repo_type="games")
# If the user wants to select a subset
if tasks != None:
valid_experiments = [
x for x in valid_experiments
if os.path.basename(x) in [os.path.basename(e) for e in tasks]
]
valid_surveys = [
x for x in valid_surveys
if os.path.basename(x) in [os.path.basename(e) for e in tasks]
]
valid_games = [
x for x in valid_games
if os.path.basename(x) in [os.path.basename(e) for e in tasks]
]
base = {
"battery_repo": battery_repo,
"experiment_repo": experiment_repo,
"survey_repo": survey_repo,
"game_repo": game_repo,
"experiments": valid_experiments,
"surveys": valid_surveys,
"games": valid_games
}
return base
def generate_experiment_web(output_dir,experiment_folder=None,survey_folder=None,games_folder=None,
make_table=True,make_index=True,make_experiments=True,make_data=True,
make_surveys=True,make_games=True):
'''get_experiment_table
Generate a table with links to preview all experiments
:param experiment_folder: folder with experiments inside
:param survey_folder: folder with surveys inside
:param games_folder: folder with games inside
:param output_dir: output folder for experiment and table html, and battery files
:param make_table: generate table.html
:param make_index: generate d3 visualization index
:param make_experiments: generate experiment preview files (linked from table and index)
:param make_data: generate json/tsv data to download
:param make_surveys: generate static files for surveys repos
:param make_games: generate static files for games repos
'''
repos=["experiments","battery"]
if make_surveys == True:
repos.append("surveys")
if make_games == True:
repos.append("games")
tmpdir = custom_battery_download(repos=repos)
if not os.path.exists(output_dir):
os.mkdir(output_dir)
if experiment_folder == None:
experiment_folder = "%s/experiments" %tmpdir
experiments = get_experiments(experiment_folder,load=True, warning=False)
experiment_tags = [x[0]["exp_id"] for x in experiments]
battery_repo = "%s/battery" %(tmpdir)
if survey_folder == None:
survey_folder = "%s/surveys" %tmpdir
if games_folder == None:
games_folder = "%s/games" %tmpdir
# If the user wants surveys and/or games, add them on to tasks
tasks = experiments
if make_surveys == True:
surveys = get_experiments(survey_folder,load=True,warning=False,repo_type="surveys")
survey_tags = [x[0]["exp_id"] for x in surveys]
tasks = experiments + surveys
if make_games == True:
games = get_experiments(games_folder,load=True,warning=False,repo_type="games")
games_tags = [x[0]["exp_id"] for x in games]
tasks = tasks + games
# Fields to keep for the table
fields = ['preview','exp_id','template',
'contributors','time',
'cognitive_atlas_task_id']
valid = pandas.DataFrame(columns=fields)
# Make a table of experiment information
for experiment in tasks:
for field in experiment[0].keys():
if field in fields:
values = experiment[0][field]
# Join lists with a comma
if field == "reference":
if values != '':
values = '<a href="%s" target="_blank">%s</a>' %(values,values)
if isinstance(values,list):
values = ",".join(values)
valid.loc[experiment[0]["exp_id"],field] = values
# Add a preview link
valid.loc[experiment[0]["exp_id"],"preview"] = '<a href="%s.html" target="_blank">DEMO</a>' %(experiment[0]["exp_id"])
# If the user wants to create the index page
if make_index == True:
output_index = os.path.abspath("%s/index.html" %output_dir)
# For each experiment, we will prepare an interactive node for the site
nodes = []
for experiment in tasks:
nodes.append('{"cluster": 1, "radius": "10", "color": colors[%s], "exp_id": "%s" }' %(choice([0,1,2]),experiment[0]["exp_id"]))
# Generate index page
index_template = get_template("%s/templates/expfactory_index.html" %get_installdir())
index_template = index_template.replace("[SUB_NODES_SUB]",",".join(nodes))
index_template = index_template.replace("[SUB_TOTAL_SUB]",str(len(nodes)))
filey = open(output_index,"wb")
filey.writelines(index_template)
filey.close()
# Update entire static directory
old_dirs = ["templates","static"]
for folder in old_dirs:
copy_to = os.path.abspath("%s/%s" %(output_dir,folder))
copy_from = "%s/battery/%s" %(tmpdir,folder)
if os.path.exists(copy_to):
shutil.rmtree(copy_to)
copy_directory(copy_from,copy_to)
# Copy the favicon to web base
shutil.copyfile("%s/battery/static/favicon.ico" %tmpdir,"%s/favicon.ico" %output_dir)
# Clear old experiments
experiment_dir = os.path.abspath("%s/static/experiments/" %output_dir)
if os.path.exists(experiment_dir):
shutil.rmtree(experiment_dir)
# Clear old surveys, copy updated valid surveys into survey directory
if make_surveys == True:
survey_dir = os.path.abspath("%s/static/surveys/" %output_dir)
if os.path.exists(survey_dir):
shutil.rmtree(survey_dir)
valid_surveys = ["%s/%s" %(survey_folder,x[0]["exp_id"]) for x in surveys]
move_experiments(valid_surveys,battery_dest=output_dir,repo_type="surveys")
# Clear old surveys, copy updated valid surveys into survey directory
if make_games == True:
games_dir = os.path.abspath("%s/static/games/" %output_dir)
if os.path.exists(games_dir):
shutil.rmtree(games_dir)
valid_games = ["%s/%s" %(games_folder,x[0]["exp_id"]) for x in games]
move_experiments(valid_games,battery_dest=output_dir,repo_type="games")
# Copy updated valid experiments into our experiment directory
valid_experiments = ["%s/%s" %(experiment_folder,x[0]["exp_id"]) for x in experiments]
move_experiments(valid_experiments,battery_dest=output_dir)
# If the user wants to make a table
if make_table == True:
table_template = get_template("%s/templates/table.html" %get_installdir())
output_table = os.path.abspath("%s/table.html" %output_dir)
# First prepare rendered table
table = '<table id="fresh-table" class="table">\n<thead>\n'
for field in fields:
table = '%s<th data-field="%s" data-sortable="true">%s</th>' %(table,field,field)
table = '%s\n</thead>\n<tbody>\n' %(table)
for row in valid.iterrows():
table = "%s<tr>\n" %(table)
for field in row[1]:
table = "%s<td>%s</td>\n" %(table,field)
table = "%s</tr>\n" %(table)
table = "%s</tbody></table>\n" %(table)
# Write the new table
table_template = table_template.replace("[[SUB_TABLE_SUB]]",table)
filey = open("%s/table.html" %output_dir,"wb")
filey.writelines(table_template)
filey.close()
if make_experiments == True:
experiments_template = get_template("%s/templates/experiments_categories.html" %get_installdir())
output_exp = os.path.abspath("%s/experiments.html" %output_dir)
if "CIRCLE_BRANCH" in os.environ.keys():
experiment_page = table_template
else:
experiment_page = get_cognitiveatlas_hierarchy(experiment_tags=experiment_tags,get_html=True)
# Write the new table
filey = open(output_exp,"wb")
filey.writelines(experiment_page)
filey.close()
# For each experiment, we will generate a demo page
for experiment in experiments:
demo_page = os.path.abspath("%s/%s.html" %(output_dir,experiment[0]["exp_id"]))
exp_template = get_experiment_html(experiment,"%s/%s" %(experiment_folder,experiment[0]["exp_id"]))
filey = open(demo_page,"wb")
filey.writelines(exp_template)
filey.close()
# if the user wants to make surveys
if make_surveys == True:
for experiment in surveys:
demo_page = os.path.abspath("%s/%s.html" %(output_dir,experiment[0]["exp_id"]))
exp_template = get_experiment_html(experiment,"%s/%s" %(survey_folder,experiment[0]["exp_id"]))
filey = open(demo_page,"wb")
filey.writelines(exp_template)
filey.close()
# if the user wants to make surveys
if make_games == True:
for experiment in games:
demo_page = os.path.abspath("%s/%s.html" %(output_dir,experiment[0]["exp_id"]))
exp_template = get_experiment_html(experiment,"%s/%s" %(games_folder,experiment[0]["exp_id"]))
filey = open(demo_page,"wb")
filey.writelines(exp_template)
filey.close()
# If the user wants to make data
if make_data == True:
data_folder = os.path.abspath("%s/data" %output_dir)
if not os.path.exists(data_folder):
os.mkdir(data_folder)
save_pretty_json("%s/expfactory-experiments.json" %(data_folder),json.loads(valid.to_json(orient="records")))
valid.to_csv("%s/expfactory-experiments.tsv" %(data_folder),sep="\t",index=None)
valid.to_pickle("%s/expfactory-experiments.pkl" %(data_folder))
