def prepare_vm(battery_dest, fields=None, vm_repo=None, vm_type="vagrant"): '''prepare_vm Prepare virtual machine to run local with vagrant, or with vagrant-aws :param battery_dest: the battery destination folder :param fields: if specified, will be added to the config.txt :param vm_repo: the expfactory-vm repo to use for templates. If not provided, one will be downloaded to a temporary directory for use. :param vm_type: one of "vagrant" or "aws" Default is "vagrant" meaning a localvirtual machine with vagrant. ''' # Download vm_repo if it hasn't been specified if not vm_repo: download_repo("vm", "%s/vm" % battery_dest) # Grab the appropriate vagrantfile template if vm_type == "aws": template = get_template("%s/vm/VagrantfileAWS" % (battery_dest)) else: template = get_template("%s/vm/VagrantfileLocal" % (battery_dest)) # If the user has custom config fields, write to file if fields != None: fields = clean_fields(fields) template = sub_template(template, "[SUB_CONFIG_SUB]", str(fields)) else: template = sub_template(template, "[SUB_CONFIG_SUB]", "None") # Change file to be a custom install template = sub_template(template, 'CUSTOM_INSTALL="False"', 'CUSTOM_INSTALL="True"') output_file = "%s/Vagrantfile" % (battery_dest) save_template(output_file, template) return template
def prepare_vm(battery_dest,fields=None,vm_repo=None,vm_type="vagrant"): '''prepare_vm Prepare virtual machine to run local with vagrant, or with vagrant-aws :param battery_dest: the battery destination folder :param fields: if specified, will be added to the config.txt :param vm_repo: the expfactory-vm repo to use for templates. If not provided, one will be downloaded to a temporary directory for use. :param vm_type: one of "vagrant" or "aws" Default is "vagrant" meaning a localvirtual machine with vagrant. ''' # Download vm_repo if it hasn't been specified if not vm_repo: download_repo("vm","%s/vm" %battery_dest) # Grab the appropriate vagrantfile template if vm_type == "aws": template = get_template("%s/vm/VagrantfileAWS" %(battery_dest)) else: template = get_template("%s/vm/VagrantfileLocal" %(battery_dest)) # If the user has custom config fields, write to file if fields != None: fields = clean_fields(fields) template = sub_template(template,"[SUB_CONFIG_SUB]",str(fields)) else: template = sub_template(template,"[SUB_CONFIG_SUB]","None") # Change file to be a custom install template = sub_template(template,'CUSTOM_INSTALL="False"','CUSTOM_INSTALL="True"') output_file = "%s/Vagrantfile" %(battery_dest) save_template(output_file,template) return template
def template_experiments(battery_dest, battery_repo, valid_experiments, template_load=None, template_exp=None, template_exp_output=None, custom_variables=None): '''template_experiments: For each valid experiment, copies the entire folder into the battery destination directory, and generates templates with appropriate paths to run them :param battery_dest: full path to destination folder of battery :param battery_repo: full path to psiturk-battery repo template :param valid_experiments: a list of full paths to experiment folders to include :param template_load: the load_experiments.js template file. If not specified, the file from the battery repo is used. :param template_exp: the exp.html template file that runs load_experiment.js. If not specified, the psiturk file from the battery repo is used. :param template_exp_output: The output file for template_exp. if not specified, the default psiturk templates/exp.html is used :param custom_variables: A dictionary of custom variables to add to templates. Keys should either be "exp" or "load", and values should be tuples with the first index the thing to sub (eg, [SUB_THIS_SUB]) and the second the substitition to make. ''' # Generate run template, make substitutions if template_load == None: template_load = "%s/static/js/load_experiments.js" % (battery_repo) if template_exp == None: template_exp = "%s/templates/exp.html" % (battery_repo) if template_exp_output == None: template_exp_output = "%s/templates/exp.html" % (battery_dest) load_template = get_template(template_load) exp_template = get_template(template_exp) valid_experiments = move_experiments(valid_experiments, battery_dest) loadstatic = get_load_static(valid_experiments) concatjs = get_concat_js(valid_experiments) timingjs = get_timing_js(valid_experiments) load_template = sub_template(load_template, "[SUB_EXPERIMENTCONCAT_SUB]", concatjs) exp_template = sub_template(exp_template, "[SUB_EXPERIMENTSTATIC_SUB]", loadstatic) load_template = sub_template(load_template, "[SUB_EXPERIMENTTIMES_SUB]", str(timingjs)) # Add custom user variables if custom_variables != None: if "exp" in custom_variables: exp_template = add_custom_variables(custom_variables["exp"], exp_template) if "load" in custom_variables: load_template = add_custom_variables(custom_variables["load"], load_template) # load experiment scripts if not os.path.exists("%s/static/js" % (battery_dest)): os.mkdir("%s/static/js" % (battery_dest)) template_output = "%s/static/js/load_experiments.js" % (battery_dest) filey = open(template_output, 'w') filey.writelines(load_template) filey.close() # exp.html template filey = open(template_exp_output, 'w') filey.writelines(exp_template) filey.close()
def template_experiments(battery_dest,battery_repo,valid_experiments,template_load=None,template_exp=None, template_exp_output=None,custom_variables=None): '''template_experiments: For each valid experiment, copies the entire folder into the battery destination directory, and generates templates with appropriate paths to run them :param battery_dest: full path to destination folder of battery :param battery_repo: full path to psiturk-battery repo template :param valid_experiments: a list of full paths to experiment folders to include :param template_load: the load_experiments.js template file. If not specified, the file from the battery repo is used. :param template_exp: the exp.html template file that runs load_experiment.js. If not specified, the psiturk file from the battery repo is used. :param template_exp_output: The output file for template_exp. if not specified, the default psiturk templates/exp.html is used :param custom_variables: A dictionary of custom variables to add to templates. Keys should either be "exp" or "load", and values should be tuples with the first index the thing to sub (eg, [SUB_THIS_SUB]) and the second the substitition to make. ''' # Generate run template, make substitutions if template_load == None: template_load = "%s/static/js/load_experiments.js" %(battery_repo) if template_exp == None: template_exp = "%s/templates/exp.html" %(battery_repo) if template_exp_output == None: template_exp_output = "%s/templates/exp.html" %(battery_dest) load_template = get_template(template_load) exp_template = get_template(template_exp) valid_experiments = move_experiments(valid_experiments,battery_dest) loadstatic = get_load_static(valid_experiments) concatjs = get_concat_js(valid_experiments) timingjs = get_timing_js(valid_experiments) load_template = sub_template(load_template,"[SUB_EXPERIMENTCONCAT_SUB]",concatjs) exp_template = sub_template(exp_template,"[SUB_EXPERIMENTSTATIC_SUB]",loadstatic) load_template = sub_template(load_template,"[SUB_EXPERIMENTTIMES_SUB]",str(timingjs)) # Add custom user variables if custom_variables != None: if "exp" in custom_variables: exp_template = add_custom_variables(custom_variables["exp"],exp_template) if "load" in custom_variables: load_template = add_custom_variables(custom_variables["load"],load_template) # load experiment scripts if not os.path.exists("%s/static/js" %(battery_dest)): os.mkdir("%s/static/js" %(battery_dest)) template_output = "%s/static/js/load_experiments.js" %(battery_dest) filey = open(template_output,'w') filey.writelines(load_template) filey.close() # exp.html template filey = open(template_exp_output,'w') filey.writelines(exp_template) filey.close()
def specify_experiments(battery_dest,experiments): '''specify_experiments Specify experiments for a Vagrantfile in an output folder :param battery_dest: destination folder for battery :param experiments: a list of experiment tags to include ''' experiments = [e.encode("utf-8") for e in experiments] vagrantfile = "%s/Vagrantfile" %(battery_dest) if not os.path.exists(vagrantfile): prepare_vm(battery_dest) template = get_template(vagrantfile) template = sub_template(template,"[SUB_EXPERIMENTS_SUB]",str(experiments)) save_template(vagrantfile,template) return template
def get_config(): '''get_config load in a dummy config file from expfactory ''' module_path = get_installdir() template = "%s/templates/config.txt" % (module_path) config = get_template(template) config = config.split(os.linesep) for l in range(len(config)): line = config[l] if len(line) > 0: if line[0] != "[": fields = [x.strip(" ") for x in line.split("=")] config[l] = {fields[0]: fields[1]} return config
def get_config(): '''get_config load in a dummy config file from expfactory ''' module_path = get_installdir() template = "%s/templates/config.txt" %(module_path) config = get_template(template) config = config.split(os.linesep) for l in range(len(config)): line = config[l] if len(line)>0: if line[0]!="[": fields = [x.strip(" ") for x in line.split("=")] config[l] = {fields[0]:fields[1]} return config
def specify_experiments(battery_dest, experiments): '''specify_experiments Specify experiments for a Vagrantfile in an output folder :param battery_dest: destination folder for battery :param experiments: a list of experiment tags to include ''' experiments = [e.encode("utf-8") for e in experiments] vagrantfile = "%s/Vagrantfile" % (battery_dest) if not os.path.exists(vagrantfile): prepare_vm(battery_dest) template = get_template(vagrantfile) template = sub_template(template, "[SUB_EXPERIMENTS_SUB]", str(experiments)) save_template(vagrantfile, template) return template
def main(): parser = get_parser() try: args = parser.parse_args() except: sys.exit(0) folder = args.folder if folder is None: folder = os.getcwd() folder = os.path.abspath(folder) survey = "%s/survey.tsv" % folder if not os.path.exists(survey): bot.error("Cannot find %s, required to generate survey." % survey) sys.exit(1) config = load_experiment(folder) html, validation = generate_survey(config=config, survey_file=survey, form_action=args.action) output = args.output if output is None: output = folder output_index = "%s/index.html" % folder if os.path.exists(output_index) and args.force is False: bot.error("%s already exists, use --force to overwrite." % output_index) sys.exit(1) bot.info("Writing output files to %s" % output_index) template = get_template('survey/index.html') template = sub_template(template, "{{html}}", html) template = sub_template(template, "{{exp_id}}", config['exp_id']) template = sub_template(template, "{{validation}}", validation) write_file(output_index, template)
def main(args, parser, subparser): folder = args.folder if folder is None: folder = os.getcwd() source = args.src[0] if source is None: bot.error('Please provide a Github https address to install.') sys.exit(1) # Is the experiment valid? cli = ExperimentValidator() valid = cli.validate(source, cleanup=False) exp_id = os.path.basename(source).replace('.git', '') if valid is True: # Local Install if os.path.exists(source): config = load_experiment(source) source = os.path.abspath(source) else: config = load_experiment("%s/%s" % (cli.tmpdir, exp_id)) source = "%s/%s" % (cli.tmpdir, exp_id) exp_id = config['exp_id'] python_module = exp_id.replace('-', '_').lower() else: bot.error('%s is not valid.' % exp_id) sys.exit(1) # Move static files to output folder dest = "%s/%s" % (folder, exp_id) bot.log("Installing %s to %s" % (exp_id, dest)) # Building container in_container = False if os.environ.get('SINGULARITY_IMAGE') is not None: in_container = True # Running, live container elif os.environ.get('EXPFACTORY_CONTAINER') is not None: in_container = True if in_container is True: # if in container, we always force args.force = True bot.log("Preparing experiment routes...") template = get_template('experiments/template.py') template = sub_template(template, '{{ exp_id }}', exp_id) template = sub_template(template, '{{ exp_id_python }}', python_module) # 1. Python blueprint views = get_viewsdir(base=args.base) view_output = "%s/%s.py" % (views, python_module) save_template(view_output, template, base=views) # 2. append to __init__ init = "%s/__init__.py" % views with open(init, 'a') as filey: filey.writelines('from .%s import *\n' % python_module) # 3. Instructions if "instructions" in config: instruct = "%s/%s.help" % (views, python_module) with open(instruct, 'w') as filey: filey.writelines(config['instructions']) if not os.path.exists(dest): os.system('mkdir -p %s' % dest) else: if args.force is False: bot.error('%s is not empty! Use --force to delete and re-create.' % folder) sys.exit(1) # We don't need to copy if experiment already there if source != dest: os.system('cp -R %s/* %s' % (source, dest))
def make_tree_from_triples(triples,output_html=False,meta_data=None,delim="\t",prune_tree=True): '''make_tree_from_triples generate a tree from a data structure with the following format :param triples: csv or pandas data frame either a file or pandas data frame with this format ..note:: id parent name 0 1 None BASE 1 trm_4a3fd79d096be trm_4a3fd79d0aec1 abductive reasoning 2 trm_4a3fd79d096e3 trm_4a3fd79d09827 abstract analogy 3 trm_4a3fd79d096f0 trm_4a3fd79d0a746 abstract knowledge 4 trm_4a3fd79d096fc 1 acoustic coding The index (0-4) is not used. The head/base node should have id 1, and no parent. All other nodes can be unique ids that you specify, with a name. :param delim: the delimiter use to separate values :param output_html: return html page instead of json data structure :param prune_tree: boolean do not include nodes that don't have task children default [True] :param meta_data [OPTIONAL]: dict if defined, should be a dictionary of dictionaries, with ..note:: key = the node id value = a dictionary of meta data. For example: { "trm_12345":{"defined_by":"squidward","score":1.2}, "node_54321":{"date":"12/15/15"}, } ''' nodes = [] if not isinstance(triples,pandas.DataFrame): triples = pandas.read_csv(relationship_table,sep="\t") # check for correct columns and circular references check_pandas_columns(df=triples,column_names=["id","name","parent"]) find_circular_reference(triples) # Generate nodes unique_nodes = triples.id.unique().tolist() print "%s unique nodes found." %(len(unique_nodes)) for node in unique_nodes: parents = triples.parent[triples.id==node].tolist() name = triples.name[triples.id==node].unique().tolist() if len(name) > 1: raise ValueError("Error, node %s is defined with multiple names." %node) meta = {} if meta_data: if node in meta_data: meta = meta_data[node] nodes.append(Node(node, parents, name[0], meta)) # Generate json graph = get_json(nodes) if prune_tree == True: walk(graph,func=do_pruning) if output_html == True: # Generate the dynamic list - we will limit to three levels into the ontology html_snippet = '' # We will save a dictionary of base (experiment) nodes experiment_nodes = dict() # Save a list of nodes without concept_parents orphan_experiment_nodes = dict() for child in graph["children"]: # For each child, we will tag base nodes with parent ids parent_ids = [child["nid"]] # This first level cannot be a base node, so we don't check. # Update the experiment_nodes lookup with experiment nodes at this level exp_nodes = [x for x in child["children"] if re.search("node_",x["nid"])] if len(exp_nodes) == 0: orphan_experiment_nodes = add_orphan_experiment_nodes(orphan_experiment_nodes,child) experiment_nodes = add_experiment_nodes(experiment_nodes,exp_nodes,parent_ids) # Do we have all base (experiment) nodes? if len(exp_nodes)==len(child["children"]): # Children do not get added to list html_snippet = '%s<a><li class="accord" id="accord_%s">%s</li></a>' %(html_snippet,str(child["nid"]),str(child["name"])) else: html_snippet = '%s<li><a id="accord_%s" class="toggle accord">%s</a>' %(html_snippet,str(child["nid"]),str(child["name"])) # Add non-children html_snippet = html_snippet + '<ul class="inner">' other_nodes = [x for x in child["children"] if not re.search("node_",x["nid"])] for other_node in other_nodes: sub_parent_ids = parent_ids[:] sub_parent_ids.append(other_node["nid"]) # Update the experiment_nodes lookup with experiment nodes at this level exp_nodes = [x for x in other_node["children"] if re.search("node_",x["nid"])] if len(exp_nodes) == 0: orphan_experiment_nodes = add_orphan_experiment_nodes(orphan_experiment_nodes,other_node) experiment_nodes = add_experiment_nodes(experiment_nodes,exp_nodes,sub_parent_ids) # Do we have all base (experiment) nodes? if len(exp_nodes)==len(other_node["children"]): html_snippet = '%s<a><li id="accord_%s" class="accord">%s</li></a>' %(html_snippet,str(other_node["nid"]),str(other_node["name"])) else: html_snippet = '%s<li><a id="accord_%s" class="toggle accord">%s</a>' %(html_snippet,str(other_node["nid"]),str(other_node["name"])) html_snippet = html_snippet + '<ul class="inner">' # Add non children last_nodes = [x for x in other_node["children"] if not re.search("node_",x["nid"])] for last_node in last_nodes: last_parent_ids = sub_parent_ids[:] last_parent_ids.append(last_node["nid"]) # One last final go to update experiment nodes exp_nodes = [x for x in last_node["children"] if re.search("node_",x["nid"])] if len(exp_nodes) == 0: orphan_experiment_nodes = add_orphan_experiment_nodes(orphan_experiment_nodes,last_node) experiment_nodes = add_experiment_nodes(experiment_nodes,exp_nodes,last_parent_ids) if len(exp_nodes)==len(last_node["children"]): html_snippet = '%s<a><li id="accord_%s" class="accord">%s</li></a>' %(html_snippet,str(last_node["nid"]),str(last_node["name"])) else: html_snippet = '%s<li><a href="#" id="accord_%s" class="toggle accord">%s</a>' %(html_snippet,str(last_node["nid"]),str(last_node["name"])) html_snippet = html_snippet + '<ul class="inner">' base_nodes = [x for x in last_node["children"] if not re.search("node_",x["nid"])] # Regardless of more layers, we generate links here for all remaining base_node links. for base_node in base_nodes: html_snippet = '%s<a><li class="accord" id="accord_%s">%s</li></a>' %(html_snippet,str(base_node["nid"]),str(base_node["name"])) base_parent_ids = last_parent_ids[:] base_parent_ids.append(base_node["nid"]) remaining_children = [] remaining_parents = [base_node] while len(remaining_parents) > 0: current_node = remaining_parents.pop(0) remaining_children = remaining_children + [x for x in current_node["children"] if re.search("node_",x["nid"])] remaining_parents = remaining_parents + [x for x in current_node["children"] if not re.search("node_",x["nid"])] experiment_nodes = add_experiment_nodes(experiment_nodes,remaining_children,base_parent_ids) html_snippet = "%s</ul></li>" %(html_snippet) html_snippet = "%s</ul></li>" %(html_snippet) html_snippet = html_snippet + "</ul></li>" # Now we will generate html for each of the experiments, and save a lookup by concept id as we go concept_lookup = dict() html_experiments = '' orphan_experiment_nodes.update(experiment_nodes) for experiment_node,node in orphan_experiment_nodes.iteritems(): # If we have meta data, present each as a little paragraph. meta_snippet='' if meta_data != None: if node["nid"] in meta_data: meta_single = meta_data[node["nid"]] for meta_tag,meta_value in meta_single.iteritems(): if meta_tag != "experiment_variables": if isinstance(meta_value,list): meta_value = ",".join(meta_value) meta_snippet = "%s<p><strong>%s</strong>: %s</p>" %(meta_snippet,meta_tag,meta_value) html_experiments = '%s<div id="exp_%s" class="panel panel-default"><div class="alert-info" style="padding-left:10px;background-color:#F2DF49">%s</div><div class="panel-body"><p>%s</p><a href="https://expfactory.github.io/%s.html" target="_blank" class="btn btn-default">Preview Experiment</a></div></div>\n' %(html_experiments,node["name"],node["name"],meta_snippet,node["name"]) for parent in node["parents"]: if str(parent) in concept_lookup: holder = concept_lookup[str(parent)] holder.append(experiment_node) concept_lookup[str(parent)] = holder else: concept_lookup[str(parent)] = [experiment_node] # If the node is an experiment and has only root parent, add to lookup if len(node["parents"])==1: if node["parents"][0]=="1": concept_lookup[node["nid"]] = [experiment_node] # All experiments concept_lookup["all_experiments"] = orphan_experiment_nodes.keys() # Plug everything into the template template = get_template("%s/templates/experiments_categories.html" %get_installdir()) template = sub_template(template,"[[SUB_LOOKUP_SUB]]",str(concept_lookup)) template = sub_template(template,"[[SUB_EXPERIMENTS_SUB]]",html_experiments) template = sub_template(template,"[[SUB_NAVIGATION_SUB]]",html_snippet) graph = template return graph
def main(args, parser, subparser): template = "build/docker/Dockerfile.template" # Full path to template is required if provided via input if args.input is not None: template = args.input template = get_template(template) # For now, only one database provided database = args.database studyid = args.studyid experiments = args.experiments branch = "-b %s" % os.environ.get("EXPFACTORY_BRANCH", "master") headless = "false" if args.headless is True: headless = "true" template = sub_template(template, "{{studyid}}", studyid) template = sub_template(template, "{{database}}", database) template = sub_template(template, "{{headless}}", headless) template = sub_template(template, "{{branch}}", branch) if args.headless is True: bot.info( "Headless build detected, you will need to generate tokens for application entry with expfactory users --new" ) library = get_library(key="name") apps = "\n" # Add local experiments to library, first preference local_installs = 0 for experiment in experiments: if os.path.exists(experiment): bot.info("local experiment %s found, validating..." % experiment) # Is the experiment valid? cli = ExperimentValidator() valid = cli.validate(experiment, cleanup=False) if valid is True: local_installs += 1 config = load_experiment(experiment) exp_id = config["exp_id"] # If we aren't building in the experiment directory, we need to copy there output_dir = "%s/%s" % ( os.path.abspath(os.path.dirname(args.output)), exp_id, ) experiment_dir = os.path.abspath(experiment) if output_dir != experiment_dir: copy_directory(experiment_dir, output_dir) config["local"] = os.path.abspath(experiment) library[exp_id] = config # Warn the user that local installs are not reproducible (from recipe) if local_installs > 0: bot.warning( "%s local installs detected: build is not reproducible without experiment folders" % local_installs ) # Build Image with Experiments for experiment in experiments: exp_id = os.path.basename(experiment) if exp_id in library: config = library[exp_id] app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id) # Here add custom build routine, should be list of lines if "install" in config: commands = "\n".join(["RUN %s " % s for x in config["install"]]).strip( "\n" ) app = "%s%s\n" % (app, commands) # The final installation step, either from Github (url) or local folder if "local" in config: app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id) app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % ( app, exp_id, ) else: app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % ( app, config["github"], ) apps = "%s%s\n" % (apps, app) else: bot.warning("%s not in library, check spelling & punctuation." % exp_id) if apps == "\n": bot.error("No valid experiments found, cancelling recipe build.") sys.exit(1) template = sub_template(template, "{{experiments}}", apps) outfile = write_file(args.output, template) bot.log("Recipe written to %s" % outfile)
def main(args,parser,subparser): template = 'build/docker/Dockerfile.template' # Full path to template is required if provided via input if args.input is not None: template = args.input template = get_template(template) # For now, only one database provided database = args.database studyid = args.studyid experiments = args.experiments branch = "-b %s" %os.environ.get('EXPFACTORY_BRANCH','master') headless = "false" if args.headless is True: headless = "true" template = sub_template(template,"{{studyid}}",studyid) template = sub_template(template,"{{database}}",database) template = sub_template(template,"{{headless}}",headless) template = sub_template(template,"{{branch}}",branch) if args.headless is True: bot.info("Headless build detected, you will need to generate tokens for application entry with expfactory users --new") library = get_library(key='name') apps = "\n" # Add local experiments to library, first preference local_installs = 0 for experiment in experiments: if os.path.exists(experiment): bot.info('local experiment %s found, validating...' %experiment) # Is the experiment valid? cli = ExperimentValidator() valid = cli.validate(experiment, cleanup=False) if valid is True: local_installs +=1 config = load_experiment(experiment) exp_id = config['exp_id'] # If we aren't building in the experiment directory, we need to copy there output_dir = "%s/%s" %(os.path.abspath(os.path.dirname(args.output)), exp_id) experiment_dir = os.path.abspath(experiment) if output_dir != experiment_dir: copy_directory(experiment_dir, output_dir) config['local'] = os.path.abspath(experiment) library[exp_id] = config # Warn the user that local installs are not reproducible (from recipe) if local_installs > 0: bot.warning("%s local installs detected: build is not reproducible without experiment folders" %local_installs) # Build Image with Experiments for experiment in experiments: exp_id = os.path.basename(experiment) if exp_id in library: config = library[exp_id] app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" %(exp_id, exp_id) # Here add custom build routine, should be list of lines if "install" in config: commands = "\n".join(["RUN %s "%s for x in config['install']]).strip('\n') app = "%s%s\n" %(app, commands) # The final installation step, either from Github (url) or local folder if "local" in config: app = "%sADD %s /scif/apps/%s\n" %(app, exp_id, exp_id) app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, exp_id) else: app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, config['github']) apps = "%s%s\n" %(apps,app) else: bot.warning('%s not in library, check spelling & punctuation.' %exp_id) if apps == "\n": bot.error('No valid experiments found, cancelling recipe build.') sys.exit(1) template = sub_template(template,"{{experiments}}",apps) outfile = write_file(args.output,template) bot.log("Recipe written to %s" %outfile)
def main(args, parser, subparser): # List of experiments is required template = get_template('build/docker/Dockerfile.template') # For now, only one database provided database = args.database studyid = args.studyid experiments = args.experiments template = sub_template(template, "{{studyid}}", studyid) template = sub_template(template, "{{database}}", database) library = get_library(key='name') apps = "\n" # Add local experiments to library, first preference local_installs = 0 for experiment in experiments: if os.path.exists(experiment): bot.info('local experiment %s found, validating...' % experiment) # Is the experiment valid? cli = ExperimentValidator() valid = cli.validate(experiment, cleanup=False) if valid is True: local_installs += 1 config = load_experiment(experiment) exp_id = config['exp_id'] # If we aren't building in the experiment directory, we need to copy there output_dir = "%s/%s" % (os.path.abspath( os.path.dirname(args.output)), exp_id) experiment_dir = os.path.abspath(experiment) if output_dir != experiment_dir: copy_directory(experiment_dir, output_dir) config['local'] = os.path.abspath(experiment) library[exp_id] = config # Warn the user that local installs are not reproducible (from recipe) if local_installs > 0: bot.warning( "%s local installs detected: build is not reproducible without experiment folders" % local_installs) # Build Image with Experiments for experiment in experiments: exp_id = os.path.basename(experiment) if exp_id in library: config = library[exp_id] app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id) # Here add custom build routine, should be list of lines if "install" in config: commands = "\n".join( ["RUN %s " % s for x in config['install']]).strip('\n') app = "%s%s\n" % (app, commands) # The final installation step, either from Github (url) or local folder if "local" in config: app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id) app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % ( app, exp_id) else: app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % ( app, config['github']) apps = "%s%s\n" % (apps, app) else: bot.warning('%s not in library, check spelling & punctuation.' % exp_id) if apps == "\n": bot.error('No valid experiments found, cancelling recipe build.') sys.exit(1) template = sub_template(template, "{{experiments}}", apps) outfile = write_file(args.output, template) bot.log("Recipe written to %s" % outfile)
def generate_experiment_web(output_dir,experiment_folder=None,survey_folder=None,games_folder=None, make_table=True,make_index=True,make_experiments=True,make_data=True, make_surveys=True,make_games=True): '''get_experiment_table Generate a table with links to preview all experiments :param experiment_folder: folder with experiments inside :param survey_folder: folder with surveys inside :param games_folder: folder with games inside :param output_dir: output folder for experiment and table html, and battery files :param make_table: generate table.html :param make_index: generate d3 visualization index :param make_experiments: generate experiment preview files (linked from table and index) :param make_data: generate json/tsv data to download :param make_surveys: generate static files for surveys repos :param make_games: generate static files for games repos ''' repos=["experiments","battery"] if make_surveys == True: repos.append("surveys") if make_games == True: repos.append("games") tmpdir = custom_battery_download(repos=repos) if not os.path.exists(output_dir): os.mkdir(output_dir) if experiment_folder == None: experiment_folder = "%s/experiments" %tmpdir experiments = get_experiments(experiment_folder,load=True, warning=False) experiment_tags = [x[0]["exp_id"] for x in experiments] battery_repo = "%s/battery" %(tmpdir) if survey_folder == None: survey_folder = "%s/surveys" %tmpdir if games_folder == None: games_folder = "%s/games" %tmpdir # If the user wants surveys and/or games, add them on to tasks tasks = experiments if make_surveys == True: surveys = get_experiments(survey_folder,load=True,warning=False,repo_type="surveys") survey_tags = [x[0]["exp_id"] for x in surveys] tasks = experiments + surveys if make_games == True: games = get_experiments(games_folder,load=True,warning=False,repo_type="games") games_tags = [x[0]["exp_id"] for x in games] tasks = tasks + games # Fields to keep for the table fields = ['preview','exp_id','template', 'contributors','time', 'cognitive_atlas_task_id'] valid = pandas.DataFrame(columns=fields) # Make a table of experiment information for experiment in tasks: for field in experiment[0].keys(): if field in fields: values = experiment[0][field] # Join lists with a comma if field == "reference": if values != '': values = '<a href="%s" target="_blank">%s</a>' %(values,values) if isinstance(values,list): values = ",".join(values) valid.loc[experiment[0]["exp_id"],field] = values # Add a preview link valid.loc[experiment[0]["exp_id"],"preview"] = '<a href="%s.html" target="_blank">DEMO</a>' %(experiment[0]["exp_id"]) # If the user wants to create the index page if make_index == True: output_index = os.path.abspath("%s/index.html" %output_dir) # For each experiment, we will prepare an interactive node for the site nodes = [] for experiment in tasks: nodes.append('{"cluster": 1, "radius": "10", "color": colors[%s], "exp_id": "%s" }' %(choice([0,1,2]),experiment[0]["exp_id"])) # Generate index page index_template = get_template("%s/templates/expfactory_index.html" %get_installdir()) index_template = index_template.replace("[SUB_NODES_SUB]",",".join(nodes)) index_template = index_template.replace("[SUB_TOTAL_SUB]",str(len(nodes))) filey = open(output_index,"wb") filey.writelines(index_template) filey.close() # Update entire static directory old_dirs = ["templates","static"] for folder in old_dirs: copy_to = os.path.abspath("%s/%s" %(output_dir,folder)) copy_from = "%s/battery/%s" %(tmpdir,folder) if os.path.exists(copy_to): shutil.rmtree(copy_to) copy_directory(copy_from,copy_to) # Copy the favicon to web base shutil.copyfile("%s/battery/static/favicon.ico" %tmpdir,"%s/favicon.ico" %output_dir) # Clear old experiments experiment_dir = os.path.abspath("%s/static/experiments/" %output_dir) if os.path.exists(experiment_dir): shutil.rmtree(experiment_dir) # Clear old surveys, copy updated valid surveys into survey directory if make_surveys == True: survey_dir = os.path.abspath("%s/static/surveys/" %output_dir) if os.path.exists(survey_dir): shutil.rmtree(survey_dir) valid_surveys = ["%s/%s" %(survey_folder,x[0]["exp_id"]) for x in surveys] move_experiments(valid_surveys,battery_dest=output_dir,repo_type="surveys") # Clear old surveys, copy updated valid surveys into survey directory if make_games == True: games_dir = os.path.abspath("%s/static/games/" %output_dir) if os.path.exists(games_dir): shutil.rmtree(games_dir) valid_games = ["%s/%s" %(games_folder,x[0]["exp_id"]) for x in games] move_experiments(valid_games,battery_dest=output_dir,repo_type="games") # Copy updated valid experiments into our experiment directory valid_experiments = ["%s/%s" %(experiment_folder,x[0]["exp_id"]) for x in experiments] move_experiments(valid_experiments,battery_dest=output_dir) # If the user wants to make a table if make_table == True: table_template = get_template("%s/templates/table.html" %get_installdir()) output_table = os.path.abspath("%s/table.html" %output_dir) # First prepare rendered table table = '<table id="fresh-table" class="table">\n<thead>\n' for field in fields: table = '%s<th data-field="%s" data-sortable="true">%s</th>' %(table,field,field) table = '%s\n</thead>\n<tbody>\n' %(table) for row in valid.iterrows(): table = "%s<tr>\n" %(table) for field in row[1]: table = "%s<td>%s</td>\n" %(table,field) table = "%s</tr>\n" %(table) table = "%s</tbody></table>\n" %(table) # Write the new table table_template = table_template.replace("[[SUB_TABLE_SUB]]",table) filey = open("%s/table.html" %output_dir,"wb") filey.writelines(table_template) filey.close() if make_experiments == True: experiments_template = get_template("%s/templates/experiments_categories.html" %get_installdir()) output_exp = os.path.abspath("%s/experiments.html" %output_dir) if "CIRCLE_BRANCH" in os.environ.keys(): experiment_page = table_template else: experiment_page = get_cognitiveatlas_hierarchy(experiment_tags=experiment_tags,get_html=True) # Write the new table filey = open(output_exp,"wb") filey.writelines(experiment_page) filey.close() # For each experiment, we will generate a demo page for experiment in experiments: demo_page = os.path.abspath("%s/%s.html" %(output_dir,experiment[0]["exp_id"])) exp_template = get_experiment_html(experiment,"%s/%s" %(experiment_folder,experiment[0]["exp_id"])) filey = open(demo_page,"wb") filey.writelines(exp_template) filey.close() # if the user wants to make surveys if make_surveys == True: for experiment in surveys: demo_page = os.path.abspath("%s/%s.html" %(output_dir,experiment[0]["exp_id"])) exp_template = get_experiment_html(experiment,"%s/%s" %(survey_folder,experiment[0]["exp_id"])) filey = open(demo_page,"wb") filey.writelines(exp_template) filey.close() # if the user wants to make surveys if make_games == True: for experiment in games: demo_page = os.path.abspath("%s/%s.html" %(output_dir,experiment[0]["exp_id"])) exp_template = get_experiment_html(experiment,"%s/%s" %(games_folder,experiment[0]["exp_id"])) filey = open(demo_page,"wb") filey.writelines(exp_template) filey.close() # If the user wants to make data if make_data == True: data_folder = os.path.abspath("%s/data" %output_dir) if not os.path.exists(data_folder): os.mkdir(data_folder) save_pretty_json("%s/expfactory-experiments.json" %(data_folder),json.loads(valid.to_json(orient="records"))) valid.to_csv("%s/expfactory-experiments.tsv" %(data_folder),sep="\t",index=None) valid.to_pickle("%s/expfactory-experiments.pkl" %(data_folder))
def make_tree_from_triples(triples, output_html=False, meta_data=None, delim="\t", prune_tree=True): '''make_tree_from_triples generate a tree from a data structure with the following format :param triples: csv or pandas data frame either a file or pandas data frame with this format ..note:: id parent name 0 1 None BASE 1 trm_4a3fd79d096be trm_4a3fd79d0aec1 abductive reasoning 2 trm_4a3fd79d096e3 trm_4a3fd79d09827 abstract analogy 3 trm_4a3fd79d096f0 trm_4a3fd79d0a746 abstract knowledge 4 trm_4a3fd79d096fc 1 acoustic coding The index (0-4) is not used. The head/base node should have id 1, and no parent. All other nodes can be unique ids that you specify, with a name. :param delim: the delimiter use to separate values :param output_html: return html page instead of json data structure :param prune_tree: boolean do not include nodes that don't have task children default [True] :param meta_data [OPTIONAL]: dict if defined, should be a dictionary of dictionaries, with ..note:: key = the node id value = a dictionary of meta data. For example: { "trm_12345":{"defined_by":"squidward","score":1.2}, "node_54321":{"date":"12/15/15"}, } ''' nodes = [] if not isinstance(triples, pandas.DataFrame): triples = pandas.read_csv(relationship_table, sep="\t") # check for correct columns and circular references check_pandas_columns(df=triples, column_names=["id", "name", "parent"]) find_circular_reference(triples) # Generate nodes unique_nodes = triples.id.unique().tolist() print("%s unique nodes found." % (len(unique_nodes))) for node in unique_nodes: parents = triples.parent[triples.id == node].tolist() name = triples.name[triples.id == node].unique().tolist() if len(name) > 1: raise ValueError("Error, node %s is defined with multiple names." % node) meta = {} if meta_data: if node in meta_data: meta = meta_data[node] nodes.append(Node(node, parents, name[0], meta)) # Generate json graph = get_json(nodes) if prune_tree == True: walk(graph, func=do_pruning) if output_html == True: # Generate the dynamic list - we will limit to three levels into the ontology html_snippet = '' # We will save a dictionary of base (experiment) nodes experiment_nodes = dict() # Save a list of nodes without concept_parents orphan_experiment_nodes = dict() for child in graph["children"]: # For each child, we will tag base nodes with parent ids parent_ids = [child["nid"]] # This first level cannot be a base node, so we don't check. # Update the experiment_nodes lookup with experiment nodes at this level exp_nodes = [ x for x in child["children"] if re.search("node_", x["nid"]) ] if len(exp_nodes) == 0: orphan_experiment_nodes = add_orphan_experiment_nodes( orphan_experiment_nodes, child) experiment_nodes = add_experiment_nodes(experiment_nodes, exp_nodes, parent_ids) # Do we have all base (experiment) nodes? if len(exp_nodes) == len(child["children"]): # Children do not get added to list html_snippet = '%s<a><li class="accord" id="accord_%s">%s</li></a>' % ( html_snippet, str(child["nid"]), str(child["name"])) else: html_snippet = '%s<li><a id="accord_%s" class="toggle accord">%s</a>' % ( html_snippet, str(child["nid"]), str(child["name"])) # Add non-children html_snippet = html_snippet + '<ul class="inner">' other_nodes = [ x for x in child["children"] if not re.search("node_", x["nid"]) ] for other_node in other_nodes: sub_parent_ids = parent_ids[:] sub_parent_ids.append(other_node["nid"]) # Update the experiment_nodes lookup with experiment nodes at this level exp_nodes = [ x for x in other_node["children"] if re.search("node_", x["nid"]) ] if len(exp_nodes) == 0: orphan_experiment_nodes = add_orphan_experiment_nodes( orphan_experiment_nodes, other_node) experiment_nodes = add_experiment_nodes( experiment_nodes, exp_nodes, sub_parent_ids) # Do we have all base (experiment) nodes? if len(exp_nodes) == len(other_node["children"]): html_snippet = '%s<a><li id="accord_%s" class="accord">%s</li></a>' % ( html_snippet, str( other_node["nid"]), str(other_node["name"])) else: html_snippet = '%s<li><a id="accord_%s" class="toggle accord">%s</a>' % ( html_snippet, str( other_node["nid"]), str(other_node["name"])) html_snippet = html_snippet + '<ul class="inner">' # Add non children last_nodes = [ x for x in other_node["children"] if not re.search("node_", x["nid"]) ] for last_node in last_nodes: last_parent_ids = sub_parent_ids[:] last_parent_ids.append(last_node["nid"]) # One last final go to update experiment nodes exp_nodes = [ x for x in last_node["children"] if re.search("node_", x["nid"]) ] if len(exp_nodes) == 0: orphan_experiment_nodes = add_orphan_experiment_nodes( orphan_experiment_nodes, last_node) experiment_nodes = add_experiment_nodes( experiment_nodes, exp_nodes, last_parent_ids) if len(exp_nodes) == len(last_node["children"]): html_snippet = '%s<a><li id="accord_%s" class="accord">%s</li></a>' % ( html_snippet, str(last_node["nid"]), str(last_node["name"])) else: html_snippet = '%s<li><a href="#" id="accord_%s" class="toggle accord">%s</a>' % ( html_snippet, str(last_node["nid"]), str(last_node["name"])) html_snippet = html_snippet + '<ul class="inner">' base_nodes = [ x for x in last_node["children"] if not re.search("node_", x["nid"]) ] # Regardless of more layers, we generate links here for all remaining base_node links. for base_node in base_nodes: html_snippet = '%s<a><li class="accord" id="accord_%s">%s</li></a>' % ( html_snippet, str(base_node["nid"]), str(base_node["name"])) base_parent_ids = last_parent_ids[:] base_parent_ids.append(base_node["nid"]) remaining_children = [] remaining_parents = [base_node] while len(remaining_parents) > 0: current_node = remaining_parents.pop(0) remaining_children = remaining_children + [ x for x in current_node["children"] if re.search("node_", x["nid"]) ] remaining_parents = remaining_parents + [ x for x in current_node["children"] if not re.search("node_", x["nid"]) ] experiment_nodes = add_experiment_nodes( experiment_nodes, remaining_children, base_parent_ids) html_snippet = "%s</ul></li>" % (html_snippet) html_snippet = "%s</ul></li>" % (html_snippet) html_snippet = html_snippet + "</ul></li>" # Now we will generate html for each of the experiments, and save a lookup by concept id as we go concept_lookup = dict() html_experiments = '' orphan_experiment_nodes.update(experiment_nodes) for experiment_node, node in orphan_experiment_nodes.items(): # If we have meta data, present each as a little paragraph. meta_snippet = '' if meta_data != None: if node["nid"] in meta_data: meta_single = meta_data[node["nid"]] for meta_tag, meta_value in meta_single.items(): if meta_tag != "experiment_variables": if isinstance(meta_value, list): meta_value = ",".join(meta_value) meta_snippet = "%s<p><strong>%s</strong>: %s</p>" % ( meta_snippet, meta_tag, meta_value) html_experiments = '%s<div id="exp_%s" class="panel panel-default"><div class="alert-info" style="padding-left:10px;background-color:#F2DF49">%s</div><div class="panel-body"><p>%s</p><a href="https://expfactory.github.io/%s.html" target="_blank" class="btn btn-default">Preview Experiment</a></div></div>\n' % ( html_experiments, node["name"], node["name"], meta_snippet, node["name"]) for parent in node["parents"]: if str(parent) in concept_lookup: holder = concept_lookup[str(parent)] holder.append(experiment_node) concept_lookup[str(parent)] = holder else: concept_lookup[str(parent)] = [experiment_node] # If the node is an experiment and has only root parent, add to lookup if len(node["parents"]) == 1: if node["parents"][0] == "1": concept_lookup[node["nid"]] = [experiment_node] # All experiments concept_lookup["all_experiments"] = orphan_experiment_nodes.keys() # Plug everything into the template template = get_template("%s/templates/experiments_categories.html" % get_installdir()) template = sub_template(template, "[[SUB_LOOKUP_SUB]]", str(concept_lookup)) template = sub_template(template, "[[SUB_EXPERIMENTS_SUB]]", html_experiments) template = sub_template(template, "[[SUB_NAVIGATION_SUB]]", html_snippet) graph = template return graph
def main(args,parser,subparser): folder = args.folder if folder is None: folder = os.getcwd() source = args.src[0] if source is None: bot.error('Please provide a Github https address to install.') sys.exit(1) # Is the experiment valid? cli = ExperimentValidator() valid = cli.validate(source, cleanup=False) exp_id = os.path.basename(source).replace('.git','') if valid is True: # Local Install if os.path.exists(source): config = load_experiment(source) source = os.path.abspath(source) else: config = load_experiment("%s/%s" %(cli.tmpdir,exp_id)) source = "%s/%s" %(cli.tmpdir,exp_id) exp_id = config['exp_id'] python_module = exp_id.replace('-','_').lower() else: bot.error('%s is not valid.' % exp_id) sys.exit(1) # Move static files to output folder dest = "%s/%s" %(folder,exp_id) bot.log("Installing %s to %s" %(exp_id, dest)) # Building container in_container = False if os.environ.get('SINGULARITY_IMAGE') is not None: in_container = True # Running, live container elif os.environ.get('EXPFACTORY_CONTAINER') is not None: in_container = True if in_container is True: # if in container, we always force args.force = True bot.log("Preparing experiment routes...") template = get_template('experiments/template.py') template = sub_template(template, '{{ exp_id }}', exp_id) template = sub_template(template, '{{ exp_id_python }}', python_module) # 1. Python blueprint views = get_viewsdir(base=args.base) view_output = "%s/%s.py" %(views, python_module) save_template(view_output, template, base=views) # 2. append to __init__ init = "%s/__init__.py" % views with open(init,'a') as filey: filey.writelines('from .%s import *\n' %python_module) # 3. Instructions if "instructions" in config: instruct = "%s/%s.help" %(views, python_module) with open(instruct,'w') as filey: filey.writelines(config['instructions']) if not os.path.exists(dest): os.system('mkdir -p %s' %dest) else: if args.force is False: bot.error('%s is not empty! Use --force to delete and re-create.' %folder) sys.exit(1) # We don't need to copy if experiment already there if source != dest: os.system('cp -R %s/* %s' %(source, dest))