def setUp(self):
self.pwd = get_installdir()
self.tmpdir = tempfile.mkdtemp()
self.experiment = os.path.abspath("%s/testing/data/test_task/" %
self.pwd)
self.config = json.load(
open("%s/testing/data/test_task/config.json" % self.pwd, "rb"))
def setUp(self):
self.pwd = get_installdir()
self.battery_folder = "%s/testing/data" % self.pwd
self.experiment = os.path.abspath("%s/testing/data/test_task/" %
self.pwd)
self.tmpdir = tempfile.mkdtemp()
with open("%s/testing/data/test_task/config.json" % self.pwd,
"r") as filey:
self.config = json.load(filey)
def setUp(self):
self.pwd = get_installdir()
self.tmpdir = tempfile.mkdtemp()
self.battery = custom_battery_download(self.tmpdir,repos=["experiments","battery","vm"])
self.battery_folder = "%s/battery"%self.tmpdir
self.experiments_folder = "%s/experiments"%self.tmpdir
self.vm_folder = "%s/vm"%self.tmpdir
self.experiment = os.path.abspath("%s/testing/data/test_task/" %self.pwd)
self.config = json.load(open("%s/testing/data/test_task/config.json" %self.pwd,"rb"))
def generate_local(battery_dest=None,
subject_id=None,
battery_repo=None,
experiment_repo=None,
experiments=None,
warning=True,
time=30):
'''generate_local deploys a local battery
will create a battery from a template and list of experiments
:param battery_dest: is the output folder for your battery. This folder MUST NOT EXIST. If not specified, a temp directory will be used
:param battery_repo: location of psiturk-battery repo to use as a template. If not specified, will be downloaded to a temporary directory
:param experiment_repo: location of a expfactory-experiments repo to check for valid experiments. If not specified, will be downloaded to a temporary directory
:param experiments: a list of experiments, meaning the "exp_id" variable in the config.json, to include. This variable also conincides with the experiment folder name.
:param subject_id: The subject id to embed in the experiment, and the name of the results file. If none is provided, a unique ID will be generated.
:param time: Maximum amount of time for battery to endure, to select experiments
'''
if battery_dest == None:
battery_dest = tempfile.mkdtemp()
shutil.rmtree(battery_dest)
# We can only generate a battery to a folder that does not exist, to be safe
if not os.path.exists(battery_dest):
base = generate_base(battery_dest=battery_dest,
tasks=experiments,
experiment_repo=experiment_repo,
battery_repo=battery_repo,
warning=warning,
add_surveys=False)
# We will output a local battery template (without psiturk)
template_exp = "%s/templates/localbattery.html" % get_installdir()
template_exp_output = "%s/index.html" % (battery_dest)
# Generate a unique id
if subject_id == None:
subject_id = uuid.uuid4()
# Add custom variable "subject ID" to the battery - will be added to data
custom_variables = dict()
custom_variables["exp"] = [("[SUB_SUBJECT_ID_SUB]", subject_id)]
custom_variables["load"] = [("[SUB_TOTALTIME_SUB]", time)]
# Fill in templates with the experiments
template_experiments(battery_dest=battery_dest,
battery_repo=base["battery_repo"],
valid_experiments=base["experiments"],
template_exp=template_exp,
template_exp_output=template_exp_output,
custom_variables=custom_variables)
return battery_dest
else:
print "Folder exists at %s, cannot generate." % (battery_dest)
def get_config():
'''get_config
load in a dummy config file from expfactory
'''
module_path = get_installdir()
template = "%s/templates/config.txt" % (module_path)
config = get_template(template)
config = config.split(os.linesep)
for l in range(len(config)):
line = config[l]
if len(line) > 0:
if line[0] != "[":
fields = [x.strip(" ") for x in line.split("=")]
config[l] = {fields[0]: fields[1]}
return config
def get_config():
'''get_config
load in a dummy config file from expfactory
'''
module_path = get_installdir()
template = "%s/templates/config.txt" %(module_path)
config = get_template(template)
config = config.split(os.linesep)
for l in range(len(config)):
line = config[l]
if len(line)>0:
if line[0]!="[":
fields = [x.strip(" ") for x in line.split("=")]
config[l] = {fields[0]:fields[1]}
return config
def generate_local(battery_dest=None,subject_id=None,battery_repo=None,experiment_repo=None,experiments=None,warning=True,time=30):
'''generate_local deploys a local battery
will create a battery from a template and list of experiments
:param battery_dest: is the output folder for your battery. This folder MUST NOT EXIST. If not specified, a temp directory will be used
:param battery_repo: location of psiturk-battery repo to use as a template. If not specified, will be downloaded to a temporary directory
:param experiment_repo: location of a expfactory-experiments repo to check for valid experiments. If not specified, will be downloaded to a temporary directory
:param experiments: a list of experiments, meaning the "exp_id" variable in the config.json, to include. This variable also conincides with the experiment folder name.
:param subject_id: The subject id to embed in the experiment, and the name of the results file. If none is provided, a unique ID will be generated.
:param time: Maximum amount of time for battery to endure, to select experiments
'''
if battery_dest == None:
battery_dest = tempfile.mkdtemp()
shutil.rmtree(battery_dest)
# We can only generate a battery to a folder that does not exist, to be safe
if not os.path.exists(battery_dest):
base = generate_base(battery_dest=battery_dest,
tasks=experiments,
experiment_repo=experiment_repo,
battery_repo=battery_repo,
warning=warning,
add_surveys=False)
# We will output a local battery template (without psiturk)
template_exp = "%s/templates/localbattery.html" %get_installdir()
template_exp_output = "%s/index.html" %(battery_dest)
# Generate a unique id
if subject_id == None:
subject_id = uuid.uuid4()
# Add custom variable "subject ID" to the battery - will be added to data
custom_variables = dict()
custom_variables["exp"] = [("[SUB_SUBJECT_ID_SUB]",subject_id)]
custom_variables["load"] = [("[SUB_TOTALTIME_SUB]",time)]
# Fill in templates with the experiments
template_experiments(battery_dest=battery_dest,
battery_repo=base["battery_repo"],
valid_experiments=base["experiments"],
template_exp=template_exp,
template_exp_output=template_exp_output,
custom_variables=custom_variables)
return battery_dest
else:
print "Folder exists at %s, cannot generate." %(battery_dest)
def get_experiment_html(experiment,
experiment_folder,
url_prefix="",
deployment="local"):
'''get_experiment_html
return the html template to test a single experiment
:param experiment: the loaded config.json for an experiment (json)
:param experiment_folder: the experiment folder, needed for reading in a survey
:param url_prefix: prefix to put before paths, in case of custom deployment
:param deployment: deployment environment, one of docker, docker-preview, or local [default]
'''
css, js = get_stylejs(experiment, url_prefix)
validation = ""
# JsPsych experiment
if experiment[0]["template"] in ["jspsych"]:
html = ""
runcode = get_experiment_run(
experiment, deployment=deployment)[experiment[0]["exp_id"]]
template_base = "experiment"
# HTML survey
elif experiment[0]["template"] in ["survey"]:
html, validation = generate_survey(experiment, experiment_folder)
runcode = ""
template_base = "survey"
# Phaser game
elif experiment[0]["template"] in ["phaser"]:
html = ""
runcode = experiment[0]["deployment_variables"]["run"]
template_base = "phaser"
exp_template = "%s/templates/%s.html" % (get_installdir(), template_base)
# Make substitutions
exp_template = "".join(open(exp_template, "r").readlines())
exp_template = sub_template(exp_template, "{{js}}", js)
exp_template = sub_template(exp_template, "{{css}}", css)
exp_template = sub_template(exp_template, "{{run}}", runcode)
exp_template = sub_template(exp_template, "{{html}}", html)
exp_template = sub_template(exp_template, "{{validation}}", validation)
exp_template = sub_template(exp_template, "{{exp_id}}",
experiment[0]["exp_id"])
return exp_template
def get_experiment_html(experiment,experiment_folder,url_prefix="",deployment="local"):
'''get_experiment_html
return the html template to test a single experiment
:param experiment: the loaded config.json for an experiment (json)
:param experiment_folder: the experiment folder, needed for reading in a survey
:param url_prefix: prefix to put before paths, in case of custom deployment
:param deployment: deployment environment, one of docker, docker-preview, or local [default]
'''
css,js = get_stylejs(experiment,url_prefix)
validation = ""
# JsPsych experiment
if experiment[0]["template"] in ["jspsych"]:
html = ""
runcode = get_experiment_run(experiment,deployment=deployment)[experiment[0]["exp_id"]]
template_base = "experiment"
# HTML survey
elif experiment[0]["template"] in ["survey"]:
html,validation = generate_survey(experiment,experiment_folder)
runcode = ""
template_base = "survey"
# Phaser game
elif experiment[0]["template"] in ["phaser"]:
html = ""
runcode = experiment[0]["deployment_variables"]["run"]
template_base = "phaser"
exp_template = "%s/templates/%s.html" %(get_installdir(),template_base)
# Make substitutions
exp_template = "".join(open(exp_template,"r").readlines())
exp_template = sub_template(exp_template,"{{js}}",js)
exp_template = sub_template(exp_template,"{{css}}",css)
exp_template = sub_template(exp_template,"{{run}}",runcode)
exp_template = sub_template(exp_template,"{{html}}",html)
exp_template = sub_template(exp_template,"{{validation}}",validation)
exp_template = sub_template(exp_template,"{{exp_id}}",experiment[0]["exp_id"])
return exp_template
def make_tree_from_triples(triples,
output_html=False,
meta_data=None,
delim="\t",
prune_tree=True):
'''make_tree_from_triples
generate a tree from a data structure with the following format
:param triples: csv or pandas data frame
either a file or pandas data frame with this format
..note::
id parent name
0 1 None BASE
1 trm_4a3fd79d096be trm_4a3fd79d0aec1 abductive reasoning
2 trm_4a3fd79d096e3 trm_4a3fd79d09827 abstract analogy
3 trm_4a3fd79d096f0 trm_4a3fd79d0a746 abstract knowledge
4 trm_4a3fd79d096fc 1 acoustic coding
The index (0-4) is not used. The head/base node should have id 1, and no parent.
All other nodes can be unique ids that you specify, with a name.
:param delim: the delimiter use to separate values
:param output_html: return html page instead of json data structure
:param prune_tree: boolean do not include nodes that don't have task children default [True]
:param meta_data [OPTIONAL]: dict
if defined, should be a dictionary of dictionaries, with
..note::
key = the node id
value = a dictionary of meta data. For example:
{
"trm_12345":{"defined_by":"squidward","score":1.2},
"node_54321":{"date":"12/15/15"},
}
'''
nodes = []
if not isinstance(triples, pandas.DataFrame):
triples = pandas.read_csv(relationship_table, sep="\t")
# check for correct columns and circular references
check_pandas_columns(df=triples, column_names=["id", "name", "parent"])
find_circular_reference(triples)
# Generate nodes
unique_nodes = triples.id.unique().tolist()
print("%s unique nodes found." % (len(unique_nodes)))
for node in unique_nodes:
parents = triples.parent[triples.id == node].tolist()
name = triples.name[triples.id == node].unique().tolist()
if len(name) > 1:
raise ValueError("Error, node %s is defined with multiple names." %
node)
meta = {}
if meta_data:
if node in meta_data:
meta = meta_data[node]
nodes.append(Node(node, parents, name[0], meta))
# Generate json
graph = get_json(nodes)
if prune_tree == True:
walk(graph, func=do_pruning)
if output_html == True:
# Generate the dynamic list - we will limit to three levels into the ontology
html_snippet = ''
# We will save a dictionary of base (experiment) nodes
experiment_nodes = dict()
# Save a list of nodes without concept_parents
orphan_experiment_nodes = dict()
for child in graph["children"]:
# For each child, we will tag base nodes with parent ids
parent_ids = [child["nid"]]
# This first level cannot be a base node, so we don't check.
# Update the experiment_nodes lookup with experiment nodes at this level
exp_nodes = [
x for x in child["children"] if re.search("node_", x["nid"])
]
if len(exp_nodes) == 0:
orphan_experiment_nodes = add_orphan_experiment_nodes(
orphan_experiment_nodes, child)
experiment_nodes = add_experiment_nodes(experiment_nodes,
exp_nodes, parent_ids)
# Do we have all base (experiment) nodes?
if len(exp_nodes) == len(child["children"]):
# Children do not get added to list
html_snippet = '%s<a><li class="accord" id="accord_%s">%s</li></a>' % (
html_snippet, str(child["nid"]), str(child["name"]))
else:
html_snippet = '%s<li><a id="accord_%s" class="toggle accord">%s</a>' % (
html_snippet, str(child["nid"]), str(child["name"]))
# Add non-children
html_snippet = html_snippet + '<ul class="inner">'
other_nodes = [
x for x in child["children"]
if not re.search("node_", x["nid"])
]
for other_node in other_nodes:
sub_parent_ids = parent_ids[:]
sub_parent_ids.append(other_node["nid"])
# Update the experiment_nodes lookup with experiment nodes at this level
exp_nodes = [
x for x in other_node["children"]
if re.search("node_", x["nid"])
]
if len(exp_nodes) == 0:
orphan_experiment_nodes = add_orphan_experiment_nodes(
orphan_experiment_nodes, other_node)
experiment_nodes = add_experiment_nodes(
experiment_nodes, exp_nodes, sub_parent_ids)
# Do we have all base (experiment) nodes?
if len(exp_nodes) == len(other_node["children"]):
html_snippet = '%s<a><li id="accord_%s" class="accord">%s</li></a>' % (
html_snippet, str(
other_node["nid"]), str(other_node["name"]))
else:
html_snippet = '%s<li><a id="accord_%s" class="toggle accord">%s</a>' % (
html_snippet, str(
other_node["nid"]), str(other_node["name"]))
html_snippet = html_snippet + '<ul class="inner">'
# Add non children
last_nodes = [
x for x in other_node["children"]
if not re.search("node_", x["nid"])
]
for last_node in last_nodes:
last_parent_ids = sub_parent_ids[:]
last_parent_ids.append(last_node["nid"])
# One last final go to update experiment nodes
exp_nodes = [
x for x in last_node["children"]
if re.search("node_", x["nid"])
]
if len(exp_nodes) == 0:
orphan_experiment_nodes = add_orphan_experiment_nodes(
orphan_experiment_nodes, last_node)
experiment_nodes = add_experiment_nodes(
experiment_nodes, exp_nodes, last_parent_ids)
if len(exp_nodes) == len(last_node["children"]):
html_snippet = '%s<a><li id="accord_%s" class="accord">%s</li></a>' % (
html_snippet, str(last_node["nid"]),
str(last_node["name"]))
else:
html_snippet = '%s<li><a href="#" id="accord_%s" class="toggle accord">%s</a>' % (
html_snippet, str(last_node["nid"]),
str(last_node["name"]))
html_snippet = html_snippet + '<ul class="inner">'
base_nodes = [
x for x in last_node["children"]
if not re.search("node_", x["nid"])
] # Regardless of more layers, we generate links here for all remaining base_node links.
for base_node in base_nodes:
html_snippet = '%s<a><li class="accord" id="accord_%s">%s</li></a>' % (
html_snippet, str(base_node["nid"]),
str(base_node["name"]))
base_parent_ids = last_parent_ids[:]
base_parent_ids.append(base_node["nid"])
remaining_children = []
remaining_parents = [base_node]
while len(remaining_parents) > 0:
current_node = remaining_parents.pop(0)
remaining_children = remaining_children + [
x for x in current_node["children"]
if re.search("node_", x["nid"])
]
remaining_parents = remaining_parents + [
x for x in current_node["children"]
if
not re.search("node_", x["nid"])
]
experiment_nodes = add_experiment_nodes(
experiment_nodes, remaining_children,
base_parent_ids)
html_snippet = "%s</ul></li>" % (html_snippet)
html_snippet = "%s</ul></li>" % (html_snippet)
html_snippet = html_snippet + "</ul></li>"
# Now we will generate html for each of the experiments, and save a lookup by concept id as we go
concept_lookup = dict()
html_experiments = ''
orphan_experiment_nodes.update(experiment_nodes)
for experiment_node, node in orphan_experiment_nodes.items():
# If we have meta data, present each as a little paragraph.
meta_snippet = ''
if meta_data != None:
if node["nid"] in meta_data:
meta_single = meta_data[node["nid"]]
for meta_tag, meta_value in meta_single.items():
if meta_tag != "experiment_variables":
if isinstance(meta_value, list):
meta_value = ",".join(meta_value)
meta_snippet = "%s<p><strong>%s</strong>: %s</p>" % (
meta_snippet, meta_tag, meta_value)
html_experiments = '%s<div id="exp_%s" class="panel panel-default"><div class="alert-info" style="padding-left:10px;background-color:#F2DF49">%s</div><div class="panel-body"><p>%s</p><a href="https://expfactory.github.io/%s.html" target="_blank" class="btn btn-default">Preview Experiment</a></div></div>\n' % (
html_experiments, node["name"], node["name"], meta_snippet,
node["name"])
for parent in node["parents"]:
if str(parent) in concept_lookup:
holder = concept_lookup[str(parent)]
holder.append(experiment_node)
concept_lookup[str(parent)] = holder
else:
concept_lookup[str(parent)] = [experiment_node]
# If the node is an experiment and has only root parent, add to lookup
if len(node["parents"]) == 1:
if node["parents"][0] == "1":
concept_lookup[node["nid"]] = [experiment_node]
# All experiments
concept_lookup["all_experiments"] = orphan_experiment_nodes.keys()
# Plug everything into the template
template = get_template("%s/templates/experiments_categories.html" %
get_installdir())
template = sub_template(template, "[[SUB_LOOKUP_SUB]]",
str(concept_lookup))
template = sub_template(template, "[[SUB_EXPERIMENTS_SUB]]",
html_experiments)
template = sub_template(template, "[[SUB_NAVIGATION_SUB]]",
html_snippet)
graph = template
return graph
def setUp(self):
self.pwd = get_installdir()
self.tmpdir = tempfile.mkdtemp()
self.experiment = os.path.abspath("%s/testing/data/test_task/" %self.pwd)
self.config = json.load(open("%s/testing/data/test_task/config.json" %self.pwd,"rb"))
def setUp(self):
self.pwd = get_installdir()
self.tmpdir = tempfile.mkdtemp()
self.battery = "%s/battery" % self.tmpdir
self.experiment = os.path.abspath("%s/testing/data/test_task/" %
self.pwd)
def setUp(self):
self.pwd = get_installdir()
self.tmpdir = tempfile.mkdtemp()
self.battery = "%s/battery"%self.tmpdir
self.experiment = os.path.abspath("%s/testing/data/test_task/" %self.pwd)
def generate_experiment_web(output_dir,experiment_folder=None,survey_folder=None,games_folder=None,
make_table=True,make_index=True,make_experiments=True,make_data=True,
make_surveys=True,make_games=True):
'''get_experiment_table
Generate a table with links to preview all experiments
:param experiment_folder: folder with experiments inside
:param survey_folder: folder with surveys inside
:param games_folder: folder with games inside
:param output_dir: output folder for experiment and table html, and battery files
:param make_table: generate table.html
:param make_index: generate d3 visualization index
:param make_experiments: generate experiment preview files (linked from table and index)
:param make_data: generate json/tsv data to download
:param make_surveys: generate static files for surveys repos
:param make_games: generate static files for games repos
'''
repos=["experiments","battery"]
if make_surveys == True:
repos.append("surveys")
if make_games == True:
repos.append("games")
tmpdir = custom_battery_download(repos=repos)
if not os.path.exists(output_dir):
os.mkdir(output_dir)
if experiment_folder == None:
experiment_folder = "%s/experiments" %tmpdir
experiments = get_experiments(experiment_folder,load=True, warning=False)
experiment_tags = [x[0]["exp_id"] for x in experiments]
battery_repo = "%s/battery" %(tmpdir)
if survey_folder == None:
survey_folder = "%s/surveys" %tmpdir
if games_folder == None:
games_folder = "%s/games" %tmpdir
# If the user wants surveys and/or games, add them on to tasks
tasks = experiments
if make_surveys == True:
surveys = get_experiments(survey_folder,load=True,warning=False,repo_type="surveys")
survey_tags = [x[0]["exp_id"] for x in surveys]
tasks = experiments + surveys
if make_games == True:
games = get_experiments(games_folder,load=True,warning=False,repo_type="games")
games_tags = [x[0]["exp_id"] for x in games]
tasks = tasks + games
# Fields to keep for the table
fields = ['preview','exp_id','template',
'contributors','time',
'cognitive_atlas_task_id']
valid = pandas.DataFrame(columns=fields)
# Make a table of experiment information
for experiment in tasks:
for field in experiment[0].keys():
if field in fields:
values = experiment[0][field]
# Join lists with a comma
if field == "reference":
if values != '':
values = '<a href="%s" target="_blank">%s</a>' %(values,values)
if isinstance(values,list):
values = ",".join(values)
valid.loc[experiment[0]["exp_id"],field] = values
# Add a preview link
valid.loc[experiment[0]["exp_id"],"preview"] = '<a href="%s.html" target="_blank">DEMO</a>' %(experiment[0]["exp_id"])
# If the user wants to create the index page
if make_index == True:
output_index = os.path.abspath("%s/index.html" %output_dir)
# For each experiment, we will prepare an interactive node for the site
nodes = []
for experiment in tasks:
nodes.append('{"cluster": 1, "radius": "10", "color": colors[%s], "exp_id": "%s" }' %(choice([0,1,2]),experiment[0]["exp_id"]))
# Generate index page
index_template = get_template("%s/templates/expfactory_index.html" %get_installdir())
index_template = index_template.replace("[SUB_NODES_SUB]",",".join(nodes))
index_template = index_template.replace("[SUB_TOTAL_SUB]",str(len(nodes)))
filey = open(output_index,"wb")
filey.writelines(index_template)
filey.close()
# Update entire static directory
old_dirs = ["templates","static"]
for folder in old_dirs:
copy_to = os.path.abspath("%s/%s" %(output_dir,folder))
copy_from = "%s/battery/%s" %(tmpdir,folder)
if os.path.exists(copy_to):
shutil.rmtree(copy_to)
copy_directory(copy_from,copy_to)
# Copy the favicon to web base
shutil.copyfile("%s/battery/static/favicon.ico" %tmpdir,"%s/favicon.ico" %output_dir)
# Clear old experiments
experiment_dir = os.path.abspath("%s/static/experiments/" %output_dir)
if os.path.exists(experiment_dir):
shutil.rmtree(experiment_dir)
# Clear old surveys, copy updated valid surveys into survey directory
if make_surveys == True:
survey_dir = os.path.abspath("%s/static/surveys/" %output_dir)
if os.path.exists(survey_dir):
shutil.rmtree(survey_dir)
valid_surveys = ["%s/%s" %(survey_folder,x[0]["exp_id"]) for x in surveys]
move_experiments(valid_surveys,battery_dest=output_dir,repo_type="surveys")
# Clear old surveys, copy updated valid surveys into survey directory
if make_games == True:
games_dir = os.path.abspath("%s/static/games/" %output_dir)
if os.path.exists(games_dir):
shutil.rmtree(games_dir)
valid_games = ["%s/%s" %(games_folder,x[0]["exp_id"]) for x in games]
move_experiments(valid_games,battery_dest=output_dir,repo_type="games")
# Copy updated valid experiments into our experiment directory
valid_experiments = ["%s/%s" %(experiment_folder,x[0]["exp_id"]) for x in experiments]
move_experiments(valid_experiments,battery_dest=output_dir)
# If the user wants to make a table
if make_table == True:
table_template = get_template("%s/templates/table.html" %get_installdir())
output_table = os.path.abspath("%s/table.html" %output_dir)
# First prepare rendered table
table = '<table id="fresh-table" class="table">\n<thead>\n'
for field in fields:
table = '%s<th data-field="%s" data-sortable="true">%s</th>' %(table,field,field)
table = '%s\n</thead>\n<tbody>\n' %(table)
for row in valid.iterrows():
table = "%s<tr>\n" %(table)
for field in row[1]:
table = "%s<td>%s</td>\n" %(table,field)
table = "%s</tr>\n" %(table)
table = "%s</tbody></table>\n" %(table)
# Write the new table
table_template = table_template.replace("[[SUB_TABLE_SUB]]",table)
filey = open("%s/table.html" %output_dir,"wb")
filey.writelines(table_template)
filey.close()
if make_experiments == True:
experiments_template = get_template("%s/templates/experiments_categories.html" %get_installdir())
output_exp = os.path.abspath("%s/experiments.html" %output_dir)
if "CIRCLE_BRANCH" in os.environ.keys():
experiment_page = table_template
else:
experiment_page = get_cognitiveatlas_hierarchy(experiment_tags=experiment_tags,get_html=True)
# Write the new table
filey = open(output_exp,"wb")
filey.writelines(experiment_page)
filey.close()
# For each experiment, we will generate a demo page
for experiment in experiments:
demo_page = os.path.abspath("%s/%s.html" %(output_dir,experiment[0]["exp_id"]))
exp_template = get_experiment_html(experiment,"%s/%s" %(experiment_folder,experiment[0]["exp_id"]))
filey = open(demo_page,"wb")
filey.writelines(exp_template)
filey.close()
# if the user wants to make surveys
if make_surveys == True:
for experiment in surveys:
demo_page = os.path.abspath("%s/%s.html" %(output_dir,experiment[0]["exp_id"]))
exp_template = get_experiment_html(experiment,"%s/%s" %(survey_folder,experiment[0]["exp_id"]))
filey = open(demo_page,"wb")
filey.writelines(exp_template)
filey.close()
# if the user wants to make surveys
if make_games == True:
for experiment in games:
demo_page = os.path.abspath("%s/%s.html" %(output_dir,experiment[0]["exp_id"]))
exp_template = get_experiment_html(experiment,"%s/%s" %(games_folder,experiment[0]["exp_id"]))
filey = open(demo_page,"wb")
filey.writelines(exp_template)
filey.close()
# If the user wants to make data
if make_data == True:
data_folder = os.path.abspath("%s/data" %output_dir)
if not os.path.exists(data_folder):
os.mkdir(data_folder)
save_pretty_json("%s/expfactory-experiments.json" %(data_folder),json.loads(valid.to_json(orient="records")))
valid.to_csv("%s/expfactory-experiments.tsv" %(data_folder),sep="\t",index=None)
valid.to_pickle("%s/expfactory-experiments.pkl" %(data_folder))
def make_tree_from_triples(triples,output_html=False,meta_data=None,delim="\t",prune_tree=True):
'''make_tree_from_triples
generate a tree from a data structure with the following format
:param triples: csv or pandas data frame
either a file or pandas data frame with this format
..note::
id parent name
0 1 None BASE
1 trm_4a3fd79d096be trm_4a3fd79d0aec1 abductive reasoning
2 trm_4a3fd79d096e3 trm_4a3fd79d09827 abstract analogy
3 trm_4a3fd79d096f0 trm_4a3fd79d0a746 abstract knowledge
4 trm_4a3fd79d096fc 1 acoustic coding
The index (0-4) is not used. The head/base node should have id 1, and no parent.
All other nodes can be unique ids that you specify, with a name.
:param delim: the delimiter use to separate values
:param output_html: return html page instead of json data structure
:param prune_tree: boolean do not include nodes that don't have task children default [True]
:param meta_data [OPTIONAL]: dict
if defined, should be a dictionary of dictionaries, with
..note::
key = the node id
value = a dictionary of meta data. For example:
{
"trm_12345":{"defined_by":"squidward","score":1.2},
"node_54321":{"date":"12/15/15"},
}
'''
nodes = []
if not isinstance(triples,pandas.DataFrame):
triples = pandas.read_csv(relationship_table,sep="\t")
# check for correct columns and circular references
check_pandas_columns(df=triples,column_names=["id","name","parent"])
find_circular_reference(triples)
# Generate nodes
unique_nodes = triples.id.unique().tolist()
print "%s unique nodes found." %(len(unique_nodes))
for node in unique_nodes:
parents = triples.parent[triples.id==node].tolist()
name = triples.name[triples.id==node].unique().tolist()
if len(name) > 1:
raise ValueError("Error, node %s is defined with multiple names." %node)
meta = {}
if meta_data:
if node in meta_data:
meta = meta_data[node]
nodes.append(Node(node, parents, name[0], meta))
# Generate json
graph = get_json(nodes)
if prune_tree == True:
walk(graph,func=do_pruning)
if output_html == True:
# Generate the dynamic list - we will limit to three levels into the ontology
html_snippet = ''
# We will save a dictionary of base (experiment) nodes
experiment_nodes = dict()
# Save a list of nodes without concept_parents
orphan_experiment_nodes = dict()
for child in graph["children"]:
# For each child, we will tag base nodes with parent ids
parent_ids = [child["nid"]]
# This first level cannot be a base node, so we don't check.
# Update the experiment_nodes lookup with experiment nodes at this level
exp_nodes = [x for x in child["children"] if re.search("node_",x["nid"])]
if len(exp_nodes) == 0:
orphan_experiment_nodes = add_orphan_experiment_nodes(orphan_experiment_nodes,child)
experiment_nodes = add_experiment_nodes(experiment_nodes,exp_nodes,parent_ids)
# Do we have all base (experiment) nodes?
if len(exp_nodes)==len(child["children"]):
# Children do not get added to list
html_snippet = '%s<a><li class="accord" id="accord_%s">%s</li></a>' %(html_snippet,str(child["nid"]),str(child["name"]))
else:
html_snippet = '%s<li><a id="accord_%s" class="toggle accord">%s</a>' %(html_snippet,str(child["nid"]),str(child["name"]))
# Add non-children
html_snippet = html_snippet + '<ul class="inner">'
other_nodes = [x for x in child["children"] if not re.search("node_",x["nid"])]
for other_node in other_nodes:
sub_parent_ids = parent_ids[:]
sub_parent_ids.append(other_node["nid"])
# Update the experiment_nodes lookup with experiment nodes at this level
exp_nodes = [x for x in other_node["children"] if re.search("node_",x["nid"])]
if len(exp_nodes) == 0:
orphan_experiment_nodes = add_orphan_experiment_nodes(orphan_experiment_nodes,other_node)
experiment_nodes = add_experiment_nodes(experiment_nodes,exp_nodes,sub_parent_ids)
# Do we have all base (experiment) nodes?
if len(exp_nodes)==len(other_node["children"]):
html_snippet = '%s<a><li id="accord_%s" class="accord">%s</li></a>' %(html_snippet,str(other_node["nid"]),str(other_node["name"]))
else:
html_snippet = '%s<li><a id="accord_%s" class="toggle accord">%s</a>' %(html_snippet,str(other_node["nid"]),str(other_node["name"]))
html_snippet = html_snippet + '<ul class="inner">'
# Add non children
last_nodes = [x for x in other_node["children"] if not re.search("node_",x["nid"])]
for last_node in last_nodes:
last_parent_ids = sub_parent_ids[:]
last_parent_ids.append(last_node["nid"])
# One last final go to update experiment nodes
exp_nodes = [x for x in last_node["children"] if re.search("node_",x["nid"])]
if len(exp_nodes) == 0:
orphan_experiment_nodes = add_orphan_experiment_nodes(orphan_experiment_nodes,last_node)
experiment_nodes = add_experiment_nodes(experiment_nodes,exp_nodes,last_parent_ids)
if len(exp_nodes)==len(last_node["children"]):
html_snippet = '%s<a><li id="accord_%s" class="accord">%s</li></a>' %(html_snippet,str(last_node["nid"]),str(last_node["name"]))
else:
html_snippet = '%s<li><a href="#" id="accord_%s" class="toggle accord">%s</a>' %(html_snippet,str(last_node["nid"]),str(last_node["name"]))
html_snippet = html_snippet + '<ul class="inner">'
base_nodes = [x for x in last_node["children"] if not re.search("node_",x["nid"])] # Regardless of more layers, we generate links here for all remaining base_node links.
for base_node in base_nodes:
html_snippet = '%s<a><li class="accord" id="accord_%s">%s</li></a>' %(html_snippet,str(base_node["nid"]),str(base_node["name"]))
base_parent_ids = last_parent_ids[:]
base_parent_ids.append(base_node["nid"])
remaining_children = []
remaining_parents = [base_node]
while len(remaining_parents) > 0:
current_node = remaining_parents.pop(0)
remaining_children = remaining_children + [x for x in current_node["children"] if re.search("node_",x["nid"])]
remaining_parents = remaining_parents + [x for x in current_node["children"] if not re.search("node_",x["nid"])]
experiment_nodes = add_experiment_nodes(experiment_nodes,remaining_children,base_parent_ids)
html_snippet = "%s</ul></li>" %(html_snippet)
html_snippet = "%s</ul></li>" %(html_snippet)
html_snippet = html_snippet + "</ul></li>"
# Now we will generate html for each of the experiments, and save a lookup by concept id as we go
concept_lookup = dict()
html_experiments = ''
orphan_experiment_nodes.update(experiment_nodes)
for experiment_node,node in orphan_experiment_nodes.iteritems():
# If we have meta data, present each as a little paragraph.
meta_snippet=''
if meta_data != None:
if node["nid"] in meta_data:
meta_single = meta_data[node["nid"]]
for meta_tag,meta_value in meta_single.iteritems():
if meta_tag != "experiment_variables":
if isinstance(meta_value,list):
meta_value = ",".join(meta_value)
meta_snippet = "%s<p><strong>%s</strong>: %s</p>" %(meta_snippet,meta_tag,meta_value)
html_experiments = '%s<div id="exp_%s" class="panel panel-default"><div class="alert-info" style="padding-left:10px;background-color:#F2DF49">%s</div><div class="panel-body"><p>%s</p><a href="https://expfactory.github.io/%s.html" target="_blank" class="btn btn-default">Preview Experiment</a></div></div>\n' %(html_experiments,node["name"],node["name"],meta_snippet,node["name"])
for parent in node["parents"]:
if str(parent) in concept_lookup:
holder = concept_lookup[str(parent)]
holder.append(experiment_node)
concept_lookup[str(parent)] = holder
else:
concept_lookup[str(parent)] = [experiment_node]
# If the node is an experiment and has only root parent, add to lookup
if len(node["parents"])==1:
if node["parents"][0]=="1":
concept_lookup[node["nid"]] = [experiment_node]
# All experiments
concept_lookup["all_experiments"] = orphan_experiment_nodes.keys()
# Plug everything into the template
template = get_template("%s/templates/experiments_categories.html" %get_installdir())
template = sub_template(template,"[[SUB_LOOKUP_SUB]]",str(concept_lookup))
template = sub_template(template,"[[SUB_EXPERIMENTS_SUB]]",html_experiments)
template = sub_template(template,"[[SUB_NAVIGATION_SUB]]",html_snippet)
graph = template
return graph
