def prepare_vm(battery_dest,fields=None,vm_repo=None,vm_type="vagrant"):
'''prepare_vm
Prepare virtual machine to run local with vagrant, or with vagrant-aws
:param battery_dest: the battery destination folder
:param fields: if specified, will be added to the config.txt
:param vm_repo: the expfactory-vm repo to use for templates. If not provided, one will be downloaded to a temporary directory for use.
:param vm_type: one of "vagrant" or "aws" Default is "vagrant" meaning a localvirtual machine with vagrant.
'''
# Download vm_repo if it hasn't been specified
if not vm_repo:
download_repo("vm","%s/vm" %battery_dest)
# Grab the appropriate vagrantfile template
if vm_type == "aws":
template = get_template("%s/vm/VagrantfileAWS" %(battery_dest))
else:
template = get_template("%s/vm/VagrantfileLocal" %(battery_dest))
# If the user has custom config fields, write to file
if fields != None:
fields = clean_fields(fields)
template = sub_template(template,"[SUB_CONFIG_SUB]",str(fields))
else:
template = sub_template(template,"[SUB_CONFIG_SUB]","None")
# Change file to be a custom install
template = sub_template(template,'CUSTOM_INSTALL="False"','CUSTOM_INSTALL="True"')
output_file = "%s/Vagrantfile" %(battery_dest)
save_template(output_file,template)
return template
def prepare_vm(battery_dest, fields=None, vm_repo=None, vm_type="vagrant"):
'''prepare_vm
Prepare virtual machine to run local with vagrant, or with vagrant-aws
:param battery_dest: the battery destination folder
:param fields: if specified, will be added to the config.txt
:param vm_repo: the expfactory-vm repo to use for templates. If not provided, one will be downloaded to a temporary directory for use.
:param vm_type: one of "vagrant" or "aws" Default is "vagrant" meaning a localvirtual machine with vagrant.
'''
# Download vm_repo if it hasn't been specified
if not vm_repo:
download_repo("vm", "%s/vm" % battery_dest)
# Grab the appropriate vagrantfile template
if vm_type == "aws":
template = get_template("%s/vm/VagrantfileAWS" % (battery_dest))
else:
template = get_template("%s/vm/VagrantfileLocal" % (battery_dest))
# If the user has custom config fields, write to file
if fields != None:
fields = clean_fields(fields)
template = sub_template(template, "[SUB_CONFIG_SUB]", str(fields))
else:
template = sub_template(template, "[SUB_CONFIG_SUB]", "None")
# Change file to be a custom install
template = sub_template(template, 'CUSTOM_INSTALL="False"',
'CUSTOM_INSTALL="True"')
output_file = "%s/Vagrantfile" % (battery_dest)
save_template(output_file, template)
return template
def template_experiments(battery_dest,
battery_repo,
valid_experiments,
template_load=None,
template_exp=None,
template_exp_output=None,
custom_variables=None):
'''template_experiments:
For each valid experiment, copies the entire folder into the battery destination directory, and generates templates with appropriate paths to run them
:param battery_dest: full path to destination folder of battery
:param battery_repo: full path to psiturk-battery repo template
:param valid_experiments: a list of full paths to experiment folders to include
:param template_load: the load_experiments.js template file. If not specified, the file from the battery repo is used.
:param template_exp: the exp.html template file that runs load_experiment.js. If not specified, the psiturk file from the battery repo is used.
:param template_exp_output: The output file for template_exp. if not specified, the default psiturk templates/exp.html is used
:param custom_variables: A dictionary of custom variables to add to templates. Keys should either be "exp" or "load", and values should be tuples with the first index the thing to sub (eg, [SUB_THIS_SUB]) and the second the substitition to make.
'''
# Generate run template, make substitutions
if template_load == None:
template_load = "%s/static/js/load_experiments.js" % (battery_repo)
if template_exp == None:
template_exp = "%s/templates/exp.html" % (battery_repo)
if template_exp_output == None:
template_exp_output = "%s/templates/exp.html" % (battery_dest)
load_template = get_template(template_load)
exp_template = get_template(template_exp)
valid_experiments = move_experiments(valid_experiments, battery_dest)
loadstatic = get_load_static(valid_experiments)
concatjs = get_concat_js(valid_experiments)
timingjs = get_timing_js(valid_experiments)
load_template = sub_template(load_template, "[SUB_EXPERIMENTCONCAT_SUB]",
concatjs)
exp_template = sub_template(exp_template, "[SUB_EXPERIMENTSTATIC_SUB]",
loadstatic)
load_template = sub_template(load_template, "[SUB_EXPERIMENTTIMES_SUB]",
str(timingjs))
# Add custom user variables
if custom_variables != None:
if "exp" in custom_variables:
exp_template = add_custom_variables(custom_variables["exp"],
exp_template)
if "load" in custom_variables:
load_template = add_custom_variables(custom_variables["load"],
load_template)
# load experiment scripts
if not os.path.exists("%s/static/js" % (battery_dest)):
os.mkdir("%s/static/js" % (battery_dest))
template_output = "%s/static/js/load_experiments.js" % (battery_dest)
filey = open(template_output, 'w')
filey.writelines(load_template)
filey.close()
# exp.html template
filey = open(template_exp_output, 'w')
filey.writelines(exp_template)
filey.close()
def template_experiments(battery_dest,battery_repo,valid_experiments,template_load=None,template_exp=None,
template_exp_output=None,custom_variables=None):
'''template_experiments:
For each valid experiment, copies the entire folder into the battery destination directory, and generates templates with appropriate paths to run them
:param battery_dest: full path to destination folder of battery
:param battery_repo: full path to psiturk-battery repo template
:param valid_experiments: a list of full paths to experiment folders to include
:param template_load: the load_experiments.js template file. If not specified, the file from the battery repo is used.
:param template_exp: the exp.html template file that runs load_experiment.js. If not specified, the psiturk file from the battery repo is used.
:param template_exp_output: The output file for template_exp. if not specified, the default psiturk templates/exp.html is used
:param custom_variables: A dictionary of custom variables to add to templates. Keys should either be "exp" or "load", and values should be tuples with the first index the thing to sub (eg, [SUB_THIS_SUB]) and the second the substitition to make.
'''
# Generate run template, make substitutions
if template_load == None:
template_load = "%s/static/js/load_experiments.js" %(battery_repo)
if template_exp == None:
template_exp = "%s/templates/exp.html" %(battery_repo)
if template_exp_output == None:
template_exp_output = "%s/templates/exp.html" %(battery_dest)
load_template = get_template(template_load)
exp_template = get_template(template_exp)
valid_experiments = move_experiments(valid_experiments,battery_dest)
loadstatic = get_load_static(valid_experiments)
concatjs = get_concat_js(valid_experiments)
timingjs = get_timing_js(valid_experiments)
load_template = sub_template(load_template,"[SUB_EXPERIMENTCONCAT_SUB]",concatjs)
exp_template = sub_template(exp_template,"[SUB_EXPERIMENTSTATIC_SUB]",loadstatic)
load_template = sub_template(load_template,"[SUB_EXPERIMENTTIMES_SUB]",str(timingjs))
# Add custom user variables
if custom_variables != None:
if "exp" in custom_variables:
exp_template = add_custom_variables(custom_variables["exp"],exp_template)
if "load" in custom_variables:
load_template = add_custom_variables(custom_variables["load"],load_template)
# load experiment scripts
if not os.path.exists("%s/static/js" %(battery_dest)):
os.mkdir("%s/static/js" %(battery_dest))
template_output = "%s/static/js/load_experiments.js" %(battery_dest)
filey = open(template_output,'w')
filey.writelines(load_template)
filey.close()
# exp.html template
filey = open(template_exp_output,'w')
filey.writelines(exp_template)
filey.close()
def add_custom_variables(custom_variables, template):
'''add_custom_variables takes a list of tuples and a template, where each tuple is a ("[TAG]","substitution") and the template is an open file with the tag.
:param custom_variables: a list of tuples (see description above)
:param template: an open file to replace "tag" with "substitute"
'''
for custom_var in custom_variables:
template = sub_template(template, custom_var[0], str(custom_var[1]))
return template
def add_custom_variables(custom_variables,template):
'''add_custom_variables takes a list of tuples and a template, where each tuple is a ("[TAG]","substitution") and the template is an open file with the tag.
:param custom_variables: a list of tuples (see description above)
:param template: an open file to replace "tag" with "substitute"
'''
for custom_var in custom_variables:
template = sub_template(template,custom_var[0],str(custom_var[1]))
return template
def main():
parser = get_parser()
try:
args = parser.parse_args()
except:
sys.exit(0)
folder = args.folder
if folder is None:
folder = os.getcwd()
folder = os.path.abspath(folder)
survey = "%s/survey.tsv" % folder
if not os.path.exists(survey):
bot.error("Cannot find %s, required to generate survey." % survey)
sys.exit(1)
config = load_experiment(folder)
html, validation = generate_survey(config=config,
survey_file=survey,
form_action=args.action)
output = args.output
if output is None:
output = folder
output_index = "%s/index.html" % folder
if os.path.exists(output_index) and args.force is False:
bot.error("%s already exists, use --force to overwrite." %
output_index)
sys.exit(1)
bot.info("Writing output files to %s" % output_index)
template = get_template('survey/index.html')
template = sub_template(template, "{{html}}", html)
template = sub_template(template, "{{exp_id}}", config['exp_id'])
template = sub_template(template, "{{validation}}", validation)
write_file(output_index, template)
def specify_experiments(battery_dest,experiments):
'''specify_experiments
Specify experiments for a Vagrantfile in an output folder
:param battery_dest: destination folder for battery
:param experiments: a list of experiment tags to include
'''
experiments = [e.encode("utf-8") for e in experiments]
vagrantfile = "%s/Vagrantfile" %(battery_dest)
if not os.path.exists(vagrantfile):
prepare_vm(battery_dest)
template = get_template(vagrantfile)
template = sub_template(template,"[SUB_EXPERIMENTS_SUB]",str(experiments))
save_template(vagrantfile,template)
return template
def specify_experiments(battery_dest, experiments):
'''specify_experiments
Specify experiments for a Vagrantfile in an output folder
:param battery_dest: destination folder for battery
:param experiments: a list of experiment tags to include
'''
experiments = [e.encode("utf-8") for e in experiments]
vagrantfile = "%s/Vagrantfile" % (battery_dest)
if not os.path.exists(vagrantfile):
prepare_vm(battery_dest)
template = get_template(vagrantfile)
template = sub_template(template, "[SUB_EXPERIMENTS_SUB]",
str(experiments))
save_template(vagrantfile, template)
return template
def get_experiment_html(experiment,
experiment_folder,
url_prefix="",
deployment="local"):
'''get_experiment_html
return the html template to test a single experiment
:param experiment: the loaded config.json for an experiment (json)
:param experiment_folder: the experiment folder, needed for reading in a survey
:param url_prefix: prefix to put before paths, in case of custom deployment
:param deployment: deployment environment, one of docker, docker-preview, or local [default]
'''
css, js = get_stylejs(experiment, url_prefix)
validation = ""
# JsPsych experiment
if experiment[0]["template"] in ["jspsych"]:
html = ""
runcode = get_experiment_run(
experiment, deployment=deployment)[experiment[0]["exp_id"]]
template_base = "experiment"
# HTML survey
elif experiment[0]["template"] in ["survey"]:
html, validation = generate_survey(experiment, experiment_folder)
runcode = ""
template_base = "survey"
# Phaser game
elif experiment[0]["template"] in ["phaser"]:
html = ""
runcode = experiment[0]["deployment_variables"]["run"]
template_base = "phaser"
exp_template = "%s/templates/%s.html" % (get_installdir(), template_base)
# Make substitutions
exp_template = "".join(open(exp_template, "r").readlines())
exp_template = sub_template(exp_template, "{{js}}", js)
exp_template = sub_template(exp_template, "{{css}}", css)
exp_template = sub_template(exp_template, "{{run}}", runcode)
exp_template = sub_template(exp_template, "{{html}}", html)
exp_template = sub_template(exp_template, "{{validation}}", validation)
exp_template = sub_template(exp_template, "{{exp_id}}",
experiment[0]["exp_id"])
return exp_template
def get_experiment_html(experiment,experiment_folder,url_prefix="",deployment="local"):
'''get_experiment_html
return the html template to test a single experiment
:param experiment: the loaded config.json for an experiment (json)
:param experiment_folder: the experiment folder, needed for reading in a survey
:param url_prefix: prefix to put before paths, in case of custom deployment
:param deployment: deployment environment, one of docker, docker-preview, or local [default]
'''
css,js = get_stylejs(experiment,url_prefix)
validation = ""
# JsPsych experiment
if experiment[0]["template"] in ["jspsych"]:
html = ""
runcode = get_experiment_run(experiment,deployment=deployment)[experiment[0]["exp_id"]]
template_base = "experiment"
# HTML survey
elif experiment[0]["template"] in ["survey"]:
html,validation = generate_survey(experiment,experiment_folder)
runcode = ""
template_base = "survey"
# Phaser game
elif experiment[0]["template"] in ["phaser"]:
html = ""
runcode = experiment[0]["deployment_variables"]["run"]
template_base = "phaser"
exp_template = "%s/templates/%s.html" %(get_installdir(),template_base)
# Make substitutions
exp_template = "".join(open(exp_template,"r").readlines())
exp_template = sub_template(exp_template,"{{js}}",js)
exp_template = sub_template(exp_template,"{{css}}",css)
exp_template = sub_template(exp_template,"{{run}}",runcode)
exp_template = sub_template(exp_template,"{{html}}",html)
exp_template = sub_template(exp_template,"{{validation}}",validation)
exp_template = sub_template(exp_template,"{{exp_id}}",experiment[0]["exp_id"])
return exp_template
def make_tree_from_triples(triples,
output_html=False,
meta_data=None,
delim="\t",
prune_tree=True):
'''make_tree_from_triples
generate a tree from a data structure with the following format
:param triples: csv or pandas data frame
either a file or pandas data frame with this format
..note::
id parent name
0 1 None BASE
1 trm_4a3fd79d096be trm_4a3fd79d0aec1 abductive reasoning
2 trm_4a3fd79d096e3 trm_4a3fd79d09827 abstract analogy
3 trm_4a3fd79d096f0 trm_4a3fd79d0a746 abstract knowledge
4 trm_4a3fd79d096fc 1 acoustic coding
The index (0-4) is not used. The head/base node should have id 1, and no parent.
All other nodes can be unique ids that you specify, with a name.
:param delim: the delimiter use to separate values
:param output_html: return html page instead of json data structure
:param prune_tree: boolean do not include nodes that don't have task children default [True]
:param meta_data [OPTIONAL]: dict
if defined, should be a dictionary of dictionaries, with
..note::
key = the node id
value = a dictionary of meta data. For example:
{
"trm_12345":{"defined_by":"squidward","score":1.2},
"node_54321":{"date":"12/15/15"},
}
'''
nodes = []
if not isinstance(triples, pandas.DataFrame):
triples = pandas.read_csv(relationship_table, sep="\t")
# check for correct columns and circular references
check_pandas_columns(df=triples, column_names=["id", "name", "parent"])
find_circular_reference(triples)
# Generate nodes
unique_nodes = triples.id.unique().tolist()
print("%s unique nodes found." % (len(unique_nodes)))
for node in unique_nodes:
parents = triples.parent[triples.id == node].tolist()
name = triples.name[triples.id == node].unique().tolist()
if len(name) > 1:
raise ValueError("Error, node %s is defined with multiple names." %
node)
meta = {}
if meta_data:
if node in meta_data:
meta = meta_data[node]
nodes.append(Node(node, parents, name[0], meta))
# Generate json
graph = get_json(nodes)
if prune_tree == True:
walk(graph, func=do_pruning)
if output_html == True:
# Generate the dynamic list - we will limit to three levels into the ontology
html_snippet = ''
# We will save a dictionary of base (experiment) nodes
experiment_nodes = dict()
# Save a list of nodes without concept_parents
orphan_experiment_nodes = dict()
for child in graph["children"]:
# For each child, we will tag base nodes with parent ids
parent_ids = [child["nid"]]
# This first level cannot be a base node, so we don't check.
# Update the experiment_nodes lookup with experiment nodes at this level
exp_nodes = [
x for x in child["children"] if re.search("node_", x["nid"])
]
if len(exp_nodes) == 0:
orphan_experiment_nodes = add_orphan_experiment_nodes(
orphan_experiment_nodes, child)
experiment_nodes = add_experiment_nodes(experiment_nodes,
exp_nodes, parent_ids)
# Do we have all base (experiment) nodes?
if len(exp_nodes) == len(child["children"]):
# Children do not get added to list
html_snippet = '%s<a><li class="accord" id="accord_%s">%s</li></a>' % (
html_snippet, str(child["nid"]), str(child["name"]))
else:
html_snippet = '%s<li><a id="accord_%s" class="toggle accord">%s</a>' % (
html_snippet, str(child["nid"]), str(child["name"]))
# Add non-children
html_snippet = html_snippet + '<ul class="inner">'
other_nodes = [
x for x in child["children"]
if not re.search("node_", x["nid"])
]
for other_node in other_nodes:
sub_parent_ids = parent_ids[:]
sub_parent_ids.append(other_node["nid"])
# Update the experiment_nodes lookup with experiment nodes at this level
exp_nodes = [
x for x in other_node["children"]
if re.search("node_", x["nid"])
]
if len(exp_nodes) == 0:
orphan_experiment_nodes = add_orphan_experiment_nodes(
orphan_experiment_nodes, other_node)
experiment_nodes = add_experiment_nodes(
experiment_nodes, exp_nodes, sub_parent_ids)
# Do we have all base (experiment) nodes?
if len(exp_nodes) == len(other_node["children"]):
html_snippet = '%s<a><li id="accord_%s" class="accord">%s</li></a>' % (
html_snippet, str(
other_node["nid"]), str(other_node["name"]))
else:
html_snippet = '%s<li><a id="accord_%s" class="toggle accord">%s</a>' % (
html_snippet, str(
other_node["nid"]), str(other_node["name"]))
html_snippet = html_snippet + '<ul class="inner">'
# Add non children
last_nodes = [
x for x in other_node["children"]
if not re.search("node_", x["nid"])
]
for last_node in last_nodes:
last_parent_ids = sub_parent_ids[:]
last_parent_ids.append(last_node["nid"])
# One last final go to update experiment nodes
exp_nodes = [
x for x in last_node["children"]
if re.search("node_", x["nid"])
]
if len(exp_nodes) == 0:
orphan_experiment_nodes = add_orphan_experiment_nodes(
orphan_experiment_nodes, last_node)
experiment_nodes = add_experiment_nodes(
experiment_nodes, exp_nodes, last_parent_ids)
if len(exp_nodes) == len(last_node["children"]):
html_snippet = '%s<a><li id="accord_%s" class="accord">%s</li></a>' % (
html_snippet, str(last_node["nid"]),
str(last_node["name"]))
else:
html_snippet = '%s<li><a href="#" id="accord_%s" class="toggle accord">%s</a>' % (
html_snippet, str(last_node["nid"]),
str(last_node["name"]))
html_snippet = html_snippet + '<ul class="inner">'
base_nodes = [
x for x in last_node["children"]
if not re.search("node_", x["nid"])
] # Regardless of more layers, we generate links here for all remaining base_node links.
for base_node in base_nodes:
html_snippet = '%s<a><li class="accord" id="accord_%s">%s</li></a>' % (
html_snippet, str(base_node["nid"]),
str(base_node["name"]))
base_parent_ids = last_parent_ids[:]
base_parent_ids.append(base_node["nid"])
remaining_children = []
remaining_parents = [base_node]
while len(remaining_parents) > 0:
current_node = remaining_parents.pop(0)
remaining_children = remaining_children + [
x for x in current_node["children"]
if re.search("node_", x["nid"])
]
remaining_parents = remaining_parents + [
x for x in current_node["children"]
if
not re.search("node_", x["nid"])
]
experiment_nodes = add_experiment_nodes(
experiment_nodes, remaining_children,
base_parent_ids)
html_snippet = "%s</ul></li>" % (html_snippet)
html_snippet = "%s</ul></li>" % (html_snippet)
html_snippet = html_snippet + "</ul></li>"
# Now we will generate html for each of the experiments, and save a lookup by concept id as we go
concept_lookup = dict()
html_experiments = ''
orphan_experiment_nodes.update(experiment_nodes)
for experiment_node, node in orphan_experiment_nodes.items():
# If we have meta data, present each as a little paragraph.
meta_snippet = ''
if meta_data != None:
if node["nid"] in meta_data:
meta_single = meta_data[node["nid"]]
for meta_tag, meta_value in meta_single.items():
if meta_tag != "experiment_variables":
if isinstance(meta_value, list):
meta_value = ",".join(meta_value)
meta_snippet = "%s<p><strong>%s</strong>: %s</p>" % (
meta_snippet, meta_tag, meta_value)
html_experiments = '%s<div id="exp_%s" class="panel panel-default"><div class="alert-info" style="padding-left:10px;background-color:#F2DF49">%s</div><div class="panel-body"><p>%s</p><a href="https://expfactory.github.io/%s.html" target="_blank" class="btn btn-default">Preview Experiment</a></div></div>\n' % (
html_experiments, node["name"], node["name"], meta_snippet,
node["name"])
for parent in node["parents"]:
if str(parent) in concept_lookup:
holder = concept_lookup[str(parent)]
holder.append(experiment_node)
concept_lookup[str(parent)] = holder
else:
concept_lookup[str(parent)] = [experiment_node]
# If the node is an experiment and has only root parent, add to lookup
if len(node["parents"]) == 1:
if node["parents"][0] == "1":
concept_lookup[node["nid"]] = [experiment_node]
# All experiments
concept_lookup["all_experiments"] = orphan_experiment_nodes.keys()
# Plug everything into the template
template = get_template("%s/templates/experiments_categories.html" %
get_installdir())
template = sub_template(template, "[[SUB_LOOKUP_SUB]]",
str(concept_lookup))
template = sub_template(template, "[[SUB_EXPERIMENTS_SUB]]",
html_experiments)
template = sub_template(template, "[[SUB_NAVIGATION_SUB]]",
html_snippet)
graph = template
return graph
def make_tree_from_triples(triples,output_html=False,meta_data=None,delim="\t",prune_tree=True):
'''make_tree_from_triples
generate a tree from a data structure with the following format
:param triples: csv or pandas data frame
either a file or pandas data frame with this format
..note::
id parent name
0 1 None BASE
1 trm_4a3fd79d096be trm_4a3fd79d0aec1 abductive reasoning
2 trm_4a3fd79d096e3 trm_4a3fd79d09827 abstract analogy
3 trm_4a3fd79d096f0 trm_4a3fd79d0a746 abstract knowledge
4 trm_4a3fd79d096fc 1 acoustic coding
The index (0-4) is not used. The head/base node should have id 1, and no parent.
All other nodes can be unique ids that you specify, with a name.
:param delim: the delimiter use to separate values
:param output_html: return html page instead of json data structure
:param prune_tree: boolean do not include nodes that don't have task children default [True]
:param meta_data [OPTIONAL]: dict
if defined, should be a dictionary of dictionaries, with
..note::
key = the node id
value = a dictionary of meta data. For example:
{
"trm_12345":{"defined_by":"squidward","score":1.2},
"node_54321":{"date":"12/15/15"},
}
'''
nodes = []
if not isinstance(triples,pandas.DataFrame):
triples = pandas.read_csv(relationship_table,sep="\t")
# check for correct columns and circular references
check_pandas_columns(df=triples,column_names=["id","name","parent"])
find_circular_reference(triples)
# Generate nodes
unique_nodes = triples.id.unique().tolist()
print "%s unique nodes found." %(len(unique_nodes))
for node in unique_nodes:
parents = triples.parent[triples.id==node].tolist()
name = triples.name[triples.id==node].unique().tolist()
if len(name) > 1:
raise ValueError("Error, node %s is defined with multiple names." %node)
meta = {}
if meta_data:
if node in meta_data:
meta = meta_data[node]
nodes.append(Node(node, parents, name[0], meta))
# Generate json
graph = get_json(nodes)
if prune_tree == True:
walk(graph,func=do_pruning)
if output_html == True:
# Generate the dynamic list - we will limit to three levels into the ontology
html_snippet = ''
# We will save a dictionary of base (experiment) nodes
experiment_nodes = dict()
# Save a list of nodes without concept_parents
orphan_experiment_nodes = dict()
for child in graph["children"]:
# For each child, we will tag base nodes with parent ids
parent_ids = [child["nid"]]
# This first level cannot be a base node, so we don't check.
# Update the experiment_nodes lookup with experiment nodes at this level
exp_nodes = [x for x in child["children"] if re.search("node_",x["nid"])]
if len(exp_nodes) == 0:
orphan_experiment_nodes = add_orphan_experiment_nodes(orphan_experiment_nodes,child)
experiment_nodes = add_experiment_nodes(experiment_nodes,exp_nodes,parent_ids)
# Do we have all base (experiment) nodes?
if len(exp_nodes)==len(child["children"]):
# Children do not get added to list
html_snippet = '%s<a><li class="accord" id="accord_%s">%s</li></a>' %(html_snippet,str(child["nid"]),str(child["name"]))
else:
html_snippet = '%s<li><a id="accord_%s" class="toggle accord">%s</a>' %(html_snippet,str(child["nid"]),str(child["name"]))
# Add non-children
html_snippet = html_snippet + '<ul class="inner">'
other_nodes = [x for x in child["children"] if not re.search("node_",x["nid"])]
for other_node in other_nodes:
sub_parent_ids = parent_ids[:]
sub_parent_ids.append(other_node["nid"])
# Update the experiment_nodes lookup with experiment nodes at this level
exp_nodes = [x for x in other_node["children"] if re.search("node_",x["nid"])]
if len(exp_nodes) == 0:
orphan_experiment_nodes = add_orphan_experiment_nodes(orphan_experiment_nodes,other_node)
experiment_nodes = add_experiment_nodes(experiment_nodes,exp_nodes,sub_parent_ids)
# Do we have all base (experiment) nodes?
if len(exp_nodes)==len(other_node["children"]):
html_snippet = '%s<a><li id="accord_%s" class="accord">%s</li></a>' %(html_snippet,str(other_node["nid"]),str(other_node["name"]))
else:
html_snippet = '%s<li><a id="accord_%s" class="toggle accord">%s</a>' %(html_snippet,str(other_node["nid"]),str(other_node["name"]))
html_snippet = html_snippet + '<ul class="inner">'
# Add non children
last_nodes = [x for x in other_node["children"] if not re.search("node_",x["nid"])]
for last_node in last_nodes:
last_parent_ids = sub_parent_ids[:]
last_parent_ids.append(last_node["nid"])
# One last final go to update experiment nodes
exp_nodes = [x for x in last_node["children"] if re.search("node_",x["nid"])]
if len(exp_nodes) == 0:
orphan_experiment_nodes = add_orphan_experiment_nodes(orphan_experiment_nodes,last_node)
experiment_nodes = add_experiment_nodes(experiment_nodes,exp_nodes,last_parent_ids)
if len(exp_nodes)==len(last_node["children"]):
html_snippet = '%s<a><li id="accord_%s" class="accord">%s</li></a>' %(html_snippet,str(last_node["nid"]),str(last_node["name"]))
else:
html_snippet = '%s<li><a href="#" id="accord_%s" class="toggle accord">%s</a>' %(html_snippet,str(last_node["nid"]),str(last_node["name"]))
html_snippet = html_snippet + '<ul class="inner">'
base_nodes = [x for x in last_node["children"] if not re.search("node_",x["nid"])] # Regardless of more layers, we generate links here for all remaining base_node links.
for base_node in base_nodes:
html_snippet = '%s<a><li class="accord" id="accord_%s">%s</li></a>' %(html_snippet,str(base_node["nid"]),str(base_node["name"]))
base_parent_ids = last_parent_ids[:]
base_parent_ids.append(base_node["nid"])
remaining_children = []
remaining_parents = [base_node]
while len(remaining_parents) > 0:
current_node = remaining_parents.pop(0)
remaining_children = remaining_children + [x for x in current_node["children"] if re.search("node_",x["nid"])]
remaining_parents = remaining_parents + [x for x in current_node["children"] if not re.search("node_",x["nid"])]
experiment_nodes = add_experiment_nodes(experiment_nodes,remaining_children,base_parent_ids)
html_snippet = "%s</ul></li>" %(html_snippet)
html_snippet = "%s</ul></li>" %(html_snippet)
html_snippet = html_snippet + "</ul></li>"
# Now we will generate html for each of the experiments, and save a lookup by concept id as we go
concept_lookup = dict()
html_experiments = ''
orphan_experiment_nodes.update(experiment_nodes)
for experiment_node,node in orphan_experiment_nodes.iteritems():
# If we have meta data, present each as a little paragraph.
meta_snippet=''
if meta_data != None:
if node["nid"] in meta_data:
meta_single = meta_data[node["nid"]]
for meta_tag,meta_value in meta_single.iteritems():
if meta_tag != "experiment_variables":
if isinstance(meta_value,list):
meta_value = ",".join(meta_value)
meta_snippet = "%s<p><strong>%s</strong>: %s</p>" %(meta_snippet,meta_tag,meta_value)
html_experiments = '%s<div id="exp_%s" class="panel panel-default"><div class="alert-info" style="padding-left:10px;background-color:#F2DF49">%s</div><div class="panel-body"><p>%s</p><a href="https://expfactory.github.io/%s.html" target="_blank" class="btn btn-default">Preview Experiment</a></div></div>\n' %(html_experiments,node["name"],node["name"],meta_snippet,node["name"])
for parent in node["parents"]:
if str(parent) in concept_lookup:
holder = concept_lookup[str(parent)]
holder.append(experiment_node)
concept_lookup[str(parent)] = holder
else:
concept_lookup[str(parent)] = [experiment_node]
# If the node is an experiment and has only root parent, add to lookup
if len(node["parents"])==1:
if node["parents"][0]=="1":
concept_lookup[node["nid"]] = [experiment_node]
# All experiments
concept_lookup["all_experiments"] = orphan_experiment_nodes.keys()
# Plug everything into the template
template = get_template("%s/templates/experiments_categories.html" %get_installdir())
template = sub_template(template,"[[SUB_LOOKUP_SUB]]",str(concept_lookup))
template = sub_template(template,"[[SUB_EXPERIMENTS_SUB]]",html_experiments)
template = sub_template(template,"[[SUB_NAVIGATION_SUB]]",html_snippet)
graph = template
return graph
def main(args, parser, subparser):
# List of experiments is required
template = get_template('build/docker/Dockerfile.template')
# For now, only one database provided
database = args.database
studyid = args.studyid
experiments = args.experiments
template = sub_template(template, "{{studyid}}", studyid)
template = sub_template(template, "{{database}}", database)
library = get_library(key='name')
apps = "\n"
# Add local experiments to library, first preference
local_installs = 0
for experiment in experiments:
if os.path.exists(experiment):
bot.info('local experiment %s found, validating...' % experiment)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(experiment, cleanup=False)
if valid is True:
local_installs += 1
config = load_experiment(experiment)
exp_id = config['exp_id']
# If we aren't building in the experiment directory, we need to copy there
output_dir = "%s/%s" % (os.path.abspath(
os.path.dirname(args.output)), exp_id)
experiment_dir = os.path.abspath(experiment)
if output_dir != experiment_dir:
copy_directory(experiment_dir, output_dir)
config['local'] = os.path.abspath(experiment)
library[exp_id] = config
# Warn the user that local installs are not reproducible (from recipe)
if local_installs > 0:
bot.warning(
"%s local installs detected: build is not reproducible without experiment folders"
% local_installs)
# Build Image with Experiments
for experiment in experiments:
exp_id = os.path.basename(experiment)
if exp_id in library:
config = library[exp_id]
app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id)
# Here add custom build routine, should be list of lines
if "install" in config:
commands = "\n".join(
["RUN %s " % s for x in config['install']]).strip('\n')
app = "%s%s\n" % (app, commands)
# The final installation step, either from Github (url) or local folder
if "local" in config:
app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id)
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app, exp_id)
else:
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app, config['github'])
apps = "%s%s\n" % (apps, app)
else:
bot.warning('%s not in library, check spelling & punctuation.' %
exp_id)
if apps == "\n":
bot.error('No valid experiments found, cancelling recipe build.')
sys.exit(1)
template = sub_template(template, "{{experiments}}", apps)
outfile = write_file(args.output, template)
bot.log("Recipe written to %s" % outfile)
def main(args, parser, subparser):
folder = args.folder
if folder is None:
folder = os.getcwd()
source = args.src[0]
if source is None:
bot.error('Please provide a Github https address to install.')
sys.exit(1)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(source, cleanup=False)
exp_id = os.path.basename(source).replace('.git', '')
if valid is True:
# Local Install
if os.path.exists(source):
config = load_experiment(source)
source = os.path.abspath(source)
else:
config = load_experiment("%s/%s" % (cli.tmpdir, exp_id))
source = "%s/%s" % (cli.tmpdir, exp_id)
exp_id = config['exp_id']
python_module = exp_id.replace('-', '_').lower()
else:
bot.error('%s is not valid.' % exp_id)
sys.exit(1)
# Move static files to output folder
dest = "%s/%s" % (folder, exp_id)
bot.log("Installing %s to %s" % (exp_id, dest))
# Building container
in_container = False
if os.environ.get('SINGULARITY_IMAGE') is not None:
in_container = True
# Running, live container
elif os.environ.get('EXPFACTORY_CONTAINER') is not None:
in_container = True
if in_container is True:
# if in container, we always force
args.force = True
bot.log("Preparing experiment routes...")
template = get_template('experiments/template.py')
template = sub_template(template, '{{ exp_id }}', exp_id)
template = sub_template(template, '{{ exp_id_python }}', python_module)
# 1. Python blueprint
views = get_viewsdir(base=args.base)
view_output = "%s/%s.py" % (views, python_module)
save_template(view_output, template, base=views)
# 2. append to __init__
init = "%s/__init__.py" % views
with open(init, 'a') as filey:
filey.writelines('from .%s import *\n' % python_module)
# 3. Instructions
if "instructions" in config:
instruct = "%s/%s.help" % (views, python_module)
with open(instruct, 'w') as filey:
filey.writelines(config['instructions'])
if not os.path.exists(dest):
os.system('mkdir -p %s' % dest)
else:
if args.force is False:
bot.error('%s is not empty! Use --force to delete and re-create.' %
folder)
sys.exit(1)
# We don't need to copy if experiment already there
if source != dest:
os.system('cp -R %s/* %s' % (source, dest))
def main(args,parser,subparser):
template = 'build/docker/Dockerfile.template'
# Full path to template is required if provided via input
if args.input is not None:
template = args.input
template = get_template(template)
# For now, only one database provided
database = args.database
studyid = args.studyid
experiments = args.experiments
branch = "-b %s" %os.environ.get('EXPFACTORY_BRANCH','master')
headless = "false"
if args.headless is True:
headless = "true"
template = sub_template(template,"{{studyid}}",studyid)
template = sub_template(template,"{{database}}",database)
template = sub_template(template,"{{headless}}",headless)
template = sub_template(template,"{{branch}}",branch)
if args.headless is True:
bot.info("Headless build detected, you will need to generate tokens for application entry with expfactory users --new")
library = get_library(key='name')
apps = "\n"
# Add local experiments to library, first preference
local_installs = 0
for experiment in experiments:
if os.path.exists(experiment):
bot.info('local experiment %s found, validating...' %experiment)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(experiment, cleanup=False)
if valid is True:
local_installs +=1
config = load_experiment(experiment)
exp_id = config['exp_id']
# If we aren't building in the experiment directory, we need to copy there
output_dir = "%s/%s" %(os.path.abspath(os.path.dirname(args.output)), exp_id)
experiment_dir = os.path.abspath(experiment)
if output_dir != experiment_dir:
copy_directory(experiment_dir, output_dir)
config['local'] = os.path.abspath(experiment)
library[exp_id] = config
# Warn the user that local installs are not reproducible (from recipe)
if local_installs > 0:
bot.warning("%s local installs detected: build is not reproducible without experiment folders" %local_installs)
# Build Image with Experiments
for experiment in experiments:
exp_id = os.path.basename(experiment)
if exp_id in library:
config = library[exp_id]
app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" %(exp_id, exp_id)
# Here add custom build routine, should be list of lines
if "install" in config:
commands = "\n".join(["RUN %s "%s for x in config['install']]).strip('\n')
app = "%s%s\n" %(app, commands)
# The final installation step, either from Github (url) or local folder
if "local" in config:
app = "%sADD %s /scif/apps/%s\n" %(app, exp_id, exp_id)
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, exp_id)
else:
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, config['github'])
apps = "%s%s\n" %(apps,app)
else:
bot.warning('%s not in library, check spelling & punctuation.' %exp_id)
if apps == "\n":
bot.error('No valid experiments found, cancelling recipe build.')
sys.exit(1)
template = sub_template(template,"{{experiments}}",apps)
outfile = write_file(args.output,template)
bot.log("Recipe written to %s" %outfile)
def main(args, parser, subparser):
template = "build/docker/Dockerfile.template"
# Full path to template is required if provided via input
if args.input is not None:
template = args.input
template = get_template(template)
# For now, only one database provided
database = args.database
studyid = args.studyid
experiments = args.experiments
branch = "-b %s" % os.environ.get("EXPFACTORY_BRANCH", "master")
headless = "false"
if args.headless is True:
headless = "true"
template = sub_template(template, "{{studyid}}", studyid)
template = sub_template(template, "{{database}}", database)
template = sub_template(template, "{{headless}}", headless)
template = sub_template(template, "{{branch}}", branch)
if args.headless is True:
bot.info(
"Headless build detected, you will need to generate tokens for application entry with expfactory users --new"
)
library = get_library(key="name")
apps = "\n"
# Add local experiments to library, first preference
local_installs = 0
for experiment in experiments:
if os.path.exists(experiment):
bot.info("local experiment %s found, validating..." % experiment)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(experiment, cleanup=False)
if valid is True:
local_installs += 1
config = load_experiment(experiment)
exp_id = config["exp_id"]
# If we aren't building in the experiment directory, we need to copy there
output_dir = "%s/%s" % (
os.path.abspath(os.path.dirname(args.output)),
exp_id,
)
experiment_dir = os.path.abspath(experiment)
if output_dir != experiment_dir:
copy_directory(experiment_dir, output_dir)
config["local"] = os.path.abspath(experiment)
library[exp_id] = config
# Warn the user that local installs are not reproducible (from recipe)
if local_installs > 0:
bot.warning(
"%s local installs detected: build is not reproducible without experiment folders"
% local_installs
)
# Build Image with Experiments
for experiment in experiments:
exp_id = os.path.basename(experiment)
if exp_id in library:
config = library[exp_id]
app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id)
# Here add custom build routine, should be list of lines
if "install" in config:
commands = "\n".join(["RUN %s " % s for x in config["install"]]).strip(
"\n"
)
app = "%s%s\n" % (app, commands)
# The final installation step, either from Github (url) or local folder
if "local" in config:
app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id)
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app,
exp_id,
)
else:
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app,
config["github"],
)
apps = "%s%s\n" % (apps, app)
else:
bot.warning("%s not in library, check spelling & punctuation." % exp_id)
if apps == "\n":
bot.error("No valid experiments found, cancelling recipe build.")
sys.exit(1)
template = sub_template(template, "{{experiments}}", apps)
outfile = write_file(args.output, template)
bot.log("Recipe written to %s" % outfile)
def main(args,parser,subparser):
folder = args.folder
if folder is None:
folder = os.getcwd()
source = args.src[0]
if source is None:
bot.error('Please provide a Github https address to install.')
sys.exit(1)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(source, cleanup=False)
exp_id = os.path.basename(source).replace('.git','')
if valid is True:
# Local Install
if os.path.exists(source):
config = load_experiment(source)
source = os.path.abspath(source)
else:
config = load_experiment("%s/%s" %(cli.tmpdir,exp_id))
source = "%s/%s" %(cli.tmpdir,exp_id)
exp_id = config['exp_id']
python_module = exp_id.replace('-','_').lower()
else:
bot.error('%s is not valid.' % exp_id)
sys.exit(1)
# Move static files to output folder
dest = "%s/%s" %(folder,exp_id)
bot.log("Installing %s to %s" %(exp_id, dest))
# Building container
in_container = False
if os.environ.get('SINGULARITY_IMAGE') is not None:
in_container = True
# Running, live container
elif os.environ.get('EXPFACTORY_CONTAINER') is not None:
in_container = True
if in_container is True:
# if in container, we always force
args.force = True
bot.log("Preparing experiment routes...")
template = get_template('experiments/template.py')
template = sub_template(template, '{{ exp_id }}', exp_id)
template = sub_template(template, '{{ exp_id_python }}', python_module)
# 1. Python blueprint
views = get_viewsdir(base=args.base)
view_output = "%s/%s.py" %(views, python_module)
save_template(view_output, template, base=views)
# 2. append to __init__
init = "%s/__init__.py" % views
with open(init,'a') as filey:
filey.writelines('from .%s import *\n' %python_module)
# 3. Instructions
if "instructions" in config:
instruct = "%s/%s.help" %(views, python_module)
with open(instruct,'w') as filey:
filey.writelines(config['instructions'])
if not os.path.exists(dest):
os.system('mkdir -p %s' %dest)
else:
if args.force is False:
bot.error('%s is not empty! Use --force to delete and re-create.' %folder)
sys.exit(1)
# We don't need to copy if experiment already there
if source != dest:
os.system('cp -R %s/* %s' %(source, dest))
