Ejemplo n.º 1
0
def prepare_vm(battery_dest, fields=None, vm_repo=None, vm_type="vagrant"):
    '''prepare_vm
    Prepare virtual machine to run local with vagrant, or with vagrant-aws
    :param battery_dest: the battery destination folder
    :param fields: if specified, will be added to the config.txt
    :param vm_repo: the expfactory-vm repo to use for templates. If not provided, one will be downloaded to a temporary directory for use.
    :param vm_type: one of "vagrant" or "aws" Default is "vagrant" meaning a localvirtual machine with vagrant.
    
    '''

    # Download vm_repo if it hasn't been specified
    if not vm_repo:
        download_repo("vm", "%s/vm" % battery_dest)

    # Grab the appropriate vagrantfile template
    if vm_type == "aws":
        template = get_template("%s/vm/VagrantfileAWS" % (battery_dest))
    else:
        template = get_template("%s/vm/VagrantfileLocal" % (battery_dest))

    # If the user has custom config fields, write to file
    if fields != None:
        fields = clean_fields(fields)
        template = sub_template(template, "[SUB_CONFIG_SUB]", str(fields))
    else:
        template = sub_template(template, "[SUB_CONFIG_SUB]", "None")

    # Change file to be a custom install
    template = sub_template(template, 'CUSTOM_INSTALL="False"',
                            'CUSTOM_INSTALL="True"')

    output_file = "%s/Vagrantfile" % (battery_dest)
    save_template(output_file, template)
    return template
Ejemplo n.º 2
0
def prepare_vm(battery_dest,fields=None,vm_repo=None,vm_type="vagrant"):
    '''prepare_vm
    Prepare virtual machine to run local with vagrant, or with vagrant-aws
    :param battery_dest: the battery destination folder
    :param fields: if specified, will be added to the config.txt
    :param vm_repo: the expfactory-vm repo to use for templates. If not provided, one will be downloaded to a temporary directory for use.
    :param vm_type: one of "vagrant" or "aws" Default is "vagrant" meaning a localvirtual machine with vagrant.
    
    '''

    # Download vm_repo if it hasn't been specified
    if not vm_repo:
        download_repo("vm","%s/vm" %battery_dest)

    # Grab the appropriate vagrantfile template
    if vm_type == "aws":
        template = get_template("%s/vm/VagrantfileAWS" %(battery_dest))
    else:
        template = get_template("%s/vm/VagrantfileLocal" %(battery_dest))

    # If the user has custom config fields, write to file
    if fields != None:
        fields = clean_fields(fields)
        template = sub_template(template,"[SUB_CONFIG_SUB]",str(fields))
    else:
        template = sub_template(template,"[SUB_CONFIG_SUB]","None")

    # Change file to be a custom install
    template = sub_template(template,'CUSTOM_INSTALL="False"','CUSTOM_INSTALL="True"')
        
    output_file = "%s/Vagrantfile" %(battery_dest)
    save_template(output_file,template)
    return template
Ejemplo n.º 3
0
def template_experiments(battery_dest,
                         battery_repo,
                         valid_experiments,
                         template_load=None,
                         template_exp=None,
                         template_exp_output=None,
                         custom_variables=None):
    '''template_experiments:
    For each valid experiment, copies the entire folder into the battery destination directory, and generates templates with appropriate paths to run them
    :param battery_dest: full path to destination folder of battery
    :param battery_repo: full path to psiturk-battery repo template
    :param valid_experiments: a list of full paths to experiment folders to include
    :param template_load: the load_experiments.js template file. If not specified, the file from the battery repo is used.
    :param template_exp: the exp.html template file that runs load_experiment.js. If not specified, the psiturk file from the battery repo is used.
    :param template_exp_output: The output file for template_exp. if not specified, the default psiturk templates/exp.html is used
    :param custom_variables: A dictionary of custom variables to add to templates. Keys should either be "exp" or "load", and values should be tuples with the first index the thing to sub (eg, [SUB_THIS_SUB]) and the second the substitition to make.
    '''
    # Generate run template, make substitutions
    if template_load == None:
        template_load = "%s/static/js/load_experiments.js" % (battery_repo)
    if template_exp == None:
        template_exp = "%s/templates/exp.html" % (battery_repo)
    if template_exp_output == None:
        template_exp_output = "%s/templates/exp.html" % (battery_dest)
    load_template = get_template(template_load)
    exp_template = get_template(template_exp)
    valid_experiments = move_experiments(valid_experiments, battery_dest)
    loadstatic = get_load_static(valid_experiments)
    concatjs = get_concat_js(valid_experiments)
    timingjs = get_timing_js(valid_experiments)
    load_template = sub_template(load_template, "[SUB_EXPERIMENTCONCAT_SUB]",
                                 concatjs)
    exp_template = sub_template(exp_template, "[SUB_EXPERIMENTSTATIC_SUB]",
                                loadstatic)
    load_template = sub_template(load_template, "[SUB_EXPERIMENTTIMES_SUB]",
                                 str(timingjs))

    # Add custom user variables
    if custom_variables != None:
        if "exp" in custom_variables:
            exp_template = add_custom_variables(custom_variables["exp"],
                                                exp_template)
        if "load" in custom_variables:
            load_template = add_custom_variables(custom_variables["load"],
                                                 load_template)

    # load experiment scripts
    if not os.path.exists("%s/static/js" % (battery_dest)):
        os.mkdir("%s/static/js" % (battery_dest))
    template_output = "%s/static/js/load_experiments.js" % (battery_dest)
    filey = open(template_output, 'w')
    filey.writelines(load_template)
    filey.close()

    # exp.html template
    filey = open(template_exp_output, 'w')
    filey.writelines(exp_template)
    filey.close()
Ejemplo n.º 4
0
def template_experiments(battery_dest,battery_repo,valid_experiments,template_load=None,template_exp=None,
                         template_exp_output=None,custom_variables=None):
    '''template_experiments:
    For each valid experiment, copies the entire folder into the battery destination directory, and generates templates with appropriate paths to run them
    :param battery_dest: full path to destination folder of battery
    :param battery_repo: full path to psiturk-battery repo template
    :param valid_experiments: a list of full paths to experiment folders to include
    :param template_load: the load_experiments.js template file. If not specified, the file from the battery repo is used.
    :param template_exp: the exp.html template file that runs load_experiment.js. If not specified, the psiturk file from the battery repo is used.
    :param template_exp_output: The output file for template_exp. if not specified, the default psiturk templates/exp.html is used
    :param custom_variables: A dictionary of custom variables to add to templates. Keys should either be "exp" or "load", and values should be tuples with the first index the thing to sub (eg, [SUB_THIS_SUB]) and the second the substitition to make.
    '''
    # Generate run template, make substitutions
    if template_load == None:
        template_load = "%s/static/js/load_experiments.js" %(battery_repo)
    if template_exp == None:
        template_exp = "%s/templates/exp.html" %(battery_repo)
    if template_exp_output == None:
        template_exp_output = "%s/templates/exp.html" %(battery_dest)
    load_template = get_template(template_load)
    exp_template = get_template(template_exp)
    valid_experiments = move_experiments(valid_experiments,battery_dest)
    loadstatic = get_load_static(valid_experiments) 
    concatjs = get_concat_js(valid_experiments) 
    timingjs = get_timing_js(valid_experiments)
    load_template = sub_template(load_template,"[SUB_EXPERIMENTCONCAT_SUB]",concatjs)
    exp_template = sub_template(exp_template,"[SUB_EXPERIMENTSTATIC_SUB]",loadstatic)
    load_template = sub_template(load_template,"[SUB_EXPERIMENTTIMES_SUB]",str(timingjs)) 

    # Add custom user variables
    if custom_variables != None:
        if "exp" in custom_variables:
            exp_template = add_custom_variables(custom_variables["exp"],exp_template)
        if "load" in custom_variables:
            load_template = add_custom_variables(custom_variables["load"],load_template)

    # load experiment scripts
    if not os.path.exists("%s/static/js" %(battery_dest)):
        os.mkdir("%s/static/js" %(battery_dest))
    template_output = "%s/static/js/load_experiments.js" %(battery_dest)
    filey = open(template_output,'w')
    filey.writelines(load_template)
    filey.close()    

    # exp.html template
    filey = open(template_exp_output,'w')
    filey.writelines(exp_template)
    filey.close()    
Ejemplo n.º 5
0
def specify_experiments(battery_dest,experiments):
    '''specify_experiments
    Specify experiments for a Vagrantfile in an output folder
    :param battery_dest: destination folder for battery
    :param experiments: a list of experiment tags to include
    '''
    experiments = [e.encode("utf-8") for e in experiments]
    vagrantfile = "%s/Vagrantfile" %(battery_dest)
    if not os.path.exists(vagrantfile):
        prepare_vm(battery_dest)
    template = get_template(vagrantfile)
    template = sub_template(template,"[SUB_EXPERIMENTS_SUB]",str(experiments))
    save_template(vagrantfile,template)
    return template
Ejemplo n.º 6
0
def get_config():
    '''get_config
    load in a dummy config file from expfactory
    '''
    module_path = get_installdir()
    template = "%s/templates/config.txt" % (module_path)
    config = get_template(template)
    config = config.split(os.linesep)
    for l in range(len(config)):
        line = config[l]
        if len(line) > 0:
            if line[0] != "[":
                fields = [x.strip(" ") for x in line.split("=")]
                config[l] = {fields[0]: fields[1]}
    return config
Ejemplo n.º 7
0
def get_config():
    '''get_config
    load in a dummy config file from expfactory
    '''
    module_path = get_installdir()
    template = "%s/templates/config.txt" %(module_path)
    config = get_template(template)
    config = config.split(os.linesep)
    for l in range(len(config)):
        line = config[l]
        if len(line)>0:
            if line[0]!="[":
                fields = [x.strip(" ") for x in line.split("=")]
                config[l] = {fields[0]:fields[1]} 
    return config
Ejemplo n.º 8
0
def specify_experiments(battery_dest, experiments):
    '''specify_experiments
    Specify experiments for a Vagrantfile in an output folder
    :param battery_dest: destination folder for battery
    :param experiments: a list of experiment tags to include
    '''
    experiments = [e.encode("utf-8") for e in experiments]
    vagrantfile = "%s/Vagrantfile" % (battery_dest)
    if not os.path.exists(vagrantfile):
        prepare_vm(battery_dest)
    template = get_template(vagrantfile)
    template = sub_template(template, "[SUB_EXPERIMENTS_SUB]",
                            str(experiments))
    save_template(vagrantfile, template)
    return template
Ejemplo n.º 9
0
def main():

    parser = get_parser()

    try:
        args = parser.parse_args()
    except:
        sys.exit(0)

    folder = args.folder
    if folder is None:
        folder = os.getcwd()
    folder = os.path.abspath(folder)

    survey = "%s/survey.tsv" % folder
    if not os.path.exists(survey):
        bot.error("Cannot find %s, required to generate survey." % survey)
        sys.exit(1)

    config = load_experiment(folder)
    html, validation = generate_survey(config=config,
                                       survey_file=survey,
                                       form_action=args.action)

    output = args.output
    if output is None:
        output = folder

    output_index = "%s/index.html" % folder

    if os.path.exists(output_index) and args.force is False:
        bot.error("%s already exists, use --force to overwrite." %
                  output_index)
        sys.exit(1)

    bot.info("Writing output files to %s" % output_index)

    template = get_template('survey/index.html')
    template = sub_template(template, "{{html}}", html)
    template = sub_template(template, "{{exp_id}}", config['exp_id'])
    template = sub_template(template, "{{validation}}", validation)

    write_file(output_index, template)
Ejemplo n.º 10
0
def main(args, parser, subparser):

    folder = args.folder
    if folder is None:
        folder = os.getcwd()

    source = args.src[0]
    if source is None:
        bot.error('Please provide a Github https address to install.')
        sys.exit(1)

    # Is the experiment valid?
    cli = ExperimentValidator()
    valid = cli.validate(source, cleanup=False)
    exp_id = os.path.basename(source).replace('.git', '')

    if valid is True:

        # Local Install
        if os.path.exists(source):
            config = load_experiment(source)
            source = os.path.abspath(source)
        else:
            config = load_experiment("%s/%s" % (cli.tmpdir, exp_id))
            source = "%s/%s" % (cli.tmpdir, exp_id)

        exp_id = config['exp_id']
        python_module = exp_id.replace('-', '_').lower()
    else:
        bot.error('%s is not valid.' % exp_id)
        sys.exit(1)

    # Move static files to output folder
    dest = "%s/%s" % (folder, exp_id)

    bot.log("Installing %s to %s" % (exp_id, dest))

    # Building container
    in_container = False
    if os.environ.get('SINGULARITY_IMAGE') is not None:
        in_container = True

    # Running, live container
    elif os.environ.get('EXPFACTORY_CONTAINER') is not None:
        in_container = True

    if in_container is True:

        # if in container, we always force
        args.force = True

        bot.log("Preparing experiment routes...")
        template = get_template('experiments/template.py')
        template = sub_template(template, '{{ exp_id }}', exp_id)
        template = sub_template(template, '{{ exp_id_python }}', python_module)

        # 1. Python blueprint
        views = get_viewsdir(base=args.base)
        view_output = "%s/%s.py" % (views, python_module)
        save_template(view_output, template, base=views)

        # 2. append to __init__
        init = "%s/__init__.py" % views
        with open(init, 'a') as filey:
            filey.writelines('from .%s import *\n' % python_module)

        # 3. Instructions
        if "instructions" in config:
            instruct = "%s/%s.help" % (views, python_module)
        with open(instruct, 'w') as filey:
            filey.writelines(config['instructions'])

    if not os.path.exists(dest):
        os.system('mkdir -p %s' % dest)
    else:
        if args.force is False:
            bot.error('%s is not empty! Use --force to delete and re-create.' %
                      folder)
            sys.exit(1)

    # We don't need to copy if experiment already there
    if source != dest:
        os.system('cp -R %s/* %s' % (source, dest))
Ejemplo n.º 11
0
def make_tree_from_triples(triples,output_html=False,meta_data=None,delim="\t",prune_tree=True):
    '''make_tree_from_triples
    generate a tree from a data structure with the following format
    :param triples: csv or pandas data frame 
        either a file or pandas data frame with this format

    ..note::
 
                             id             parent                          name
        0                     1               None                          BASE
        1     trm_4a3fd79d096be  trm_4a3fd79d0aec1           abductive reasoning
        2     trm_4a3fd79d096e3  trm_4a3fd79d09827              abstract analogy
        3     trm_4a3fd79d096f0  trm_4a3fd79d0a746            abstract knowledge
        4     trm_4a3fd79d096fc                  1               acoustic coding
    
        The index (0-4) is not used. The head/base node should have id 1, and no parent.
        All other nodes can be unique ids that you specify, with a name.

    :param delim: the delimiter use to separate values
    :param output_html: return html page instead of json data structure
    :param prune_tree: boolean do not include nodes that don't have task children default [True]
    :param meta_data [OPTIONAL]: dict 
          if defined, should be a dictionary of dictionaries, with

    ..note::

          key = the node id
          value = a dictionary of meta data. For example:

          {
            "trm_12345":{"defined_by":"squidward","score":1.2},
            "node_54321":{"date":"12/15/15"},
          }
    
    '''
    nodes = []

    if not isinstance(triples,pandas.DataFrame):
        triples = pandas.read_csv(relationship_table,sep="\t")

    # check for correct columns and circular references
    check_pandas_columns(df=triples,column_names=["id","name","parent"])
    find_circular_reference(triples)

    # Generate nodes
    unique_nodes = triples.id.unique().tolist()
    print "%s unique nodes found." %(len(unique_nodes))
    for node in unique_nodes:
        parents = triples.parent[triples.id==node].tolist()
        name = triples.name[triples.id==node].unique().tolist()
        if len(name) > 1:
            raise ValueError("Error, node %s is defined with multiple names." %node)
        meta = {}
        if meta_data:
           if node in meta_data:
               meta = meta_data[node]
        nodes.append(Node(node, parents, name[0], meta))

    # Generate json
    graph = get_json(nodes)

    if prune_tree == True:
        walk(graph,func=do_pruning)

    if output_html == True:
        # Generate the dynamic list - we will limit to three levels into the ontology
        html_snippet = ''
        # We will save a dictionary of base (experiment) nodes
        experiment_nodes = dict()
        # Save a list of nodes without concept_parents
        orphan_experiment_nodes = dict()
        for child in graph["children"]:
            # For each child, we will tag base nodes with parent ids
            parent_ids = [child["nid"]]
            # This first level cannot be a base node, so we don't check.
            # Update the experiment_nodes lookup with experiment nodes at this level
            exp_nodes = [x for x in child["children"] if re.search("node_",x["nid"])]
            if len(exp_nodes) == 0:
                orphan_experiment_nodes = add_orphan_experiment_nodes(orphan_experiment_nodes,child)
            experiment_nodes = add_experiment_nodes(experiment_nodes,exp_nodes,parent_ids)
            # Do we have all base (experiment) nodes?
            if len(exp_nodes)==len(child["children"]):
                # Children do not get added to list
                html_snippet = '%s<a><li class="accord" id="accord_%s">%s</li></a>' %(html_snippet,str(child["nid"]),str(child["name"]))                
            else:
                html_snippet = '%s<li><a id="accord_%s" class="toggle accord">%s</a>' %(html_snippet,str(child["nid"]),str(child["name"]))
                # Add non-children 
                html_snippet = html_snippet + '<ul class="inner">' 
                other_nodes = [x for x in child["children"] if not re.search("node_",x["nid"])]
                for other_node in other_nodes:
                    sub_parent_ids = parent_ids[:]
                    sub_parent_ids.append(other_node["nid"])
                    # Update the experiment_nodes lookup with experiment nodes at this level
                    exp_nodes = [x for x in other_node["children"] if re.search("node_",x["nid"])]
                    if len(exp_nodes) == 0:
                        orphan_experiment_nodes = add_orphan_experiment_nodes(orphan_experiment_nodes,other_node)
                    experiment_nodes = add_experiment_nodes(experiment_nodes,exp_nodes,sub_parent_ids)
                    # Do we have all base (experiment) nodes?
                    if len(exp_nodes)==len(other_node["children"]):
                        html_snippet = '%s<a><li id="accord_%s" class="accord">%s</li></a>' %(html_snippet,str(other_node["nid"]),str(other_node["name"]))
                    else:
                        html_snippet = '%s<li><a id="accord_%s" class="toggle accord">%s</a>' %(html_snippet,str(other_node["nid"]),str(other_node["name"]))
                        html_snippet = html_snippet + '<ul class="inner">' 
                        # Add non children
                        last_nodes = [x for x in other_node["children"] if not re.search("node_",x["nid"])]     
                        for last_node in last_nodes:
                            last_parent_ids = sub_parent_ids[:]
                            last_parent_ids.append(last_node["nid"])
                            # One last final go to update experiment nodes
                            exp_nodes = [x for x in last_node["children"] if re.search("node_",x["nid"])]
                            if len(exp_nodes) == 0:
                                orphan_experiment_nodes = add_orphan_experiment_nodes(orphan_experiment_nodes,last_node)
                            experiment_nodes = add_experiment_nodes(experiment_nodes,exp_nodes,last_parent_ids)
                            if len(exp_nodes)==len(last_node["children"]):
                                html_snippet = '%s<a><li id="accord_%s" class="accord">%s</li></a>' %(html_snippet,str(last_node["nid"]),str(last_node["name"]))
                            else:
                                html_snippet = '%s<li><a href="#" id="accord_%s" class="toggle accord">%s</a>' %(html_snippet,str(last_node["nid"]),str(last_node["name"]))
                                html_snippet = html_snippet + '<ul class="inner">' 
                                base_nodes = [x for x in last_node["children"] if not re.search("node_",x["nid"])]             # Regardless of more layers, we generate links here for all remaining base_node links.
                                for base_node in base_nodes:
                                    html_snippet = '%s<a><li class="accord" id="accord_%s">%s</li></a>' %(html_snippet,str(base_node["nid"]),str(base_node["name"]))
                                    base_parent_ids = last_parent_ids[:]
                                    base_parent_ids.append(base_node["nid"])
                                    remaining_children = []
                                    remaining_parents = [base_node]
                                    while len(remaining_parents) > 0:
                                        current_node = remaining_parents.pop(0)
                                        remaining_children = remaining_children + [x for x in current_node["children"] if re.search("node_",x["nid"])]
                                        remaining_parents = remaining_parents + [x for x in current_node["children"] if not re.search("node_",x["nid"])]
                                    experiment_nodes = add_experiment_nodes(experiment_nodes,remaining_children,base_parent_ids)
                                html_snippet = "%s</ul></li>" %(html_snippet)
                        html_snippet = "%s</ul></li>" %(html_snippet)
                html_snippet = html_snippet + "</ul></li>"     
           
        # Now we will generate html for each of the experiments, and save a lookup by concept id as we go
        concept_lookup = dict()
        html_experiments = ''
        orphan_experiment_nodes.update(experiment_nodes)
        for experiment_node,node in orphan_experiment_nodes.iteritems():
            # If we have meta data, present each as a little paragraph.
            meta_snippet=''
            if meta_data != None:
                if node["nid"] in meta_data:
                    meta_single = meta_data[node["nid"]]
                    for meta_tag,meta_value in meta_single.iteritems():
                        if meta_tag != "experiment_variables":
                            if isinstance(meta_value,list):
                                meta_value = ",".join(meta_value)
                            meta_snippet = "%s<p><strong>%s</strong>: %s</p>" %(meta_snippet,meta_tag,meta_value)
            html_experiments = '%s<div id="exp_%s" class="panel panel-default"><div class="alert-info" style="padding-left:10px;background-color:#F2DF49">%s</div><div class="panel-body"><p>%s</p><a href="https://expfactory.github.io/%s.html" target="_blank" class="btn btn-default">Preview Experiment</a></div></div>\n' %(html_experiments,node["name"],node["name"],meta_snippet,node["name"])    
            for parent in node["parents"]:
                if str(parent) in concept_lookup:
                    holder = concept_lookup[str(parent)]
                    holder.append(experiment_node)
                    concept_lookup[str(parent)] = holder
                else:
                    concept_lookup[str(parent)] = [experiment_node]
            # If the node is an experiment and has only root parent, add to lookup
            if len(node["parents"])==1:
                if node["parents"][0]=="1":
                    concept_lookup[node["nid"]] = [experiment_node]
        # All experiments
        concept_lookup["all_experiments"] = orphan_experiment_nodes.keys()

        # Plug everything into the template
        template = get_template("%s/templates/experiments_categories.html" %get_installdir())
        template = sub_template(template,"[[SUB_LOOKUP_SUB]]",str(concept_lookup))
        template = sub_template(template,"[[SUB_EXPERIMENTS_SUB]]",html_experiments)
        template = sub_template(template,"[[SUB_NAVIGATION_SUB]]",html_snippet)           
        graph = template        
    return graph
Ejemplo n.º 12
0
def main(args, parser, subparser):

    template = "build/docker/Dockerfile.template"

    # Full path to template is required if provided via input
    if args.input is not None:
        template = args.input

    template = get_template(template)

    # For now, only one database provided
    database = args.database
    studyid = args.studyid
    experiments = args.experiments
    branch = "-b %s" % os.environ.get("EXPFACTORY_BRANCH", "master")

    headless = "false"
    if args.headless is True:
        headless = "true"

    template = sub_template(template, "{{studyid}}", studyid)
    template = sub_template(template, "{{database}}", database)
    template = sub_template(template, "{{headless}}", headless)
    template = sub_template(template, "{{branch}}", branch)

    if args.headless is True:
        bot.info(
            "Headless build detected, you will need to generate tokens for application entry with expfactory users --new"
        )

    library = get_library(key="name")

    apps = "\n"

    # Add local experiments to library, first preference
    local_installs = 0
    for experiment in experiments:
        if os.path.exists(experiment):

            bot.info("local experiment %s found, validating..." % experiment)

            # Is the experiment valid?
            cli = ExperimentValidator()
            valid = cli.validate(experiment, cleanup=False)

            if valid is True:
                local_installs += 1
                config = load_experiment(experiment)
                exp_id = config["exp_id"]

                # If we aren't building in the experiment directory, we need to copy there
                output_dir = "%s/%s" % (
                    os.path.abspath(os.path.dirname(args.output)),
                    exp_id,
                )
                experiment_dir = os.path.abspath(experiment)
                if output_dir != experiment_dir:
                    copy_directory(experiment_dir, output_dir)

                config["local"] = os.path.abspath(experiment)
                library[exp_id] = config

    # Warn the user that local installs are not reproducible (from recipe)
    if local_installs > 0:
        bot.warning(
            "%s local installs detected: build is not reproducible without experiment folders"
            % local_installs
        )

    # Build Image with Experiments
    for experiment in experiments:
        exp_id = os.path.basename(experiment)
        if exp_id in library:
            config = library[exp_id]

            app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id)

            # Here add custom build routine, should be list of lines
            if "install" in config:
                commands = "\n".join(["RUN %s " % s for x in config["install"]]).strip(
                    "\n"
                )
                app = "%s%s\n" % (app, commands)

            # The final installation step, either from Github (url) or local folder
            if "local" in config:
                app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id)
                app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
                    app,
                    exp_id,
                )
            else:
                app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
                    app,
                    config["github"],
                )
            apps = "%s%s\n" % (apps, app)

        else:
            bot.warning("%s not in library, check spelling & punctuation." % exp_id)

    if apps == "\n":
        bot.error("No valid experiments found, cancelling recipe build.")
        sys.exit(1)

    template = sub_template(template, "{{experiments}}", apps)
    outfile = write_file(args.output, template)
    bot.log("Recipe written to %s" % outfile)
Ejemplo n.º 13
0
def main(args,parser,subparser):

    template = 'build/docker/Dockerfile.template'

    # Full path to template is required if provided via input
    if args.input is not None:
        template = args.input

    template = get_template(template)
    
    # For now, only one database provided
    database = args.database
    studyid = args.studyid
    experiments = args.experiments
    branch = "-b %s" %os.environ.get('EXPFACTORY_BRANCH','master')

    headless = "false"
    if args.headless is True:
        headless = "true"

    template = sub_template(template,"{{studyid}}",studyid)
    template = sub_template(template,"{{database}}",database)
    template = sub_template(template,"{{headless}}",headless)
    template = sub_template(template,"{{branch}}",branch)

    if args.headless is True:
        bot.info("Headless build detected, you will need to generate tokens for application entry with expfactory users --new")

    library = get_library(key='name')

    apps = "\n"

    # Add local experiments to library, first preference
    local_installs = 0
    for experiment in experiments:
        if os.path.exists(experiment):

            bot.info('local experiment %s found, validating...' %experiment)

            # Is the experiment valid?
            cli = ExperimentValidator()
            valid = cli.validate(experiment, cleanup=False)

            if valid is True:
                local_installs +=1
                config = load_experiment(experiment)
                exp_id = config['exp_id']

                # If we aren't building in the experiment directory, we need to copy there
                output_dir = "%s/%s" %(os.path.abspath(os.path.dirname(args.output)), exp_id)
                experiment_dir = os.path.abspath(experiment)
                if output_dir != experiment_dir:
                    copy_directory(experiment_dir, output_dir)

                config['local'] = os.path.abspath(experiment)
                library[exp_id] = config


    # Warn the user that local installs are not reproducible (from recipe)
    if local_installs > 0:
        bot.warning("%s local installs detected: build is not reproducible without experiment folders" %local_installs)

    # Build Image with Experiments
    for experiment in experiments:
        exp_id = os.path.basename(experiment)
        if exp_id in library:
            config = library[exp_id]

            app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" %(exp_id, exp_id)

            # Here add custom build routine, should be list of lines
            if "install" in config:
                commands = "\n".join(["RUN %s "%s for x in config['install']]).strip('\n')
                app = "%s%s\n" %(app, commands)

            # The final installation step, either from Github (url) or local folder
            if "local" in config:
                app = "%sADD %s /scif/apps/%s\n" %(app, exp_id, exp_id)
                app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, exp_id)
            else:
                app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, config['github'])  
            apps = "%s%s\n" %(apps,app)


        else:
            bot.warning('%s not in library, check spelling & punctuation.' %exp_id)

    if apps == "\n":
        bot.error('No valid experiments found, cancelling recipe build.')
        sys.exit(1)

    template = sub_template(template,"{{experiments}}",apps)
    outfile = write_file(args.output,template)
    bot.log("Recipe written to %s" %outfile)
Ejemplo n.º 14
0
def main(args, parser, subparser):

    # List of experiments is required
    template = get_template('build/docker/Dockerfile.template')

    # For now, only one database provided
    database = args.database
    studyid = args.studyid
    experiments = args.experiments

    template = sub_template(template, "{{studyid}}", studyid)
    template = sub_template(template, "{{database}}", database)

    library = get_library(key='name')

    apps = "\n"

    # Add local experiments to library, first preference
    local_installs = 0
    for experiment in experiments:
        if os.path.exists(experiment):

            bot.info('local experiment %s found, validating...' % experiment)

            # Is the experiment valid?
            cli = ExperimentValidator()
            valid = cli.validate(experiment, cleanup=False)

            if valid is True:
                local_installs += 1
                config = load_experiment(experiment)
                exp_id = config['exp_id']

                # If we aren't building in the experiment directory, we need to copy there
                output_dir = "%s/%s" % (os.path.abspath(
                    os.path.dirname(args.output)), exp_id)
                experiment_dir = os.path.abspath(experiment)
                if output_dir != experiment_dir:
                    copy_directory(experiment_dir, output_dir)

                config['local'] = os.path.abspath(experiment)
                library[exp_id] = config

    # Warn the user that local installs are not reproducible (from recipe)
    if local_installs > 0:
        bot.warning(
            "%s local installs detected: build is not reproducible without experiment folders"
            % local_installs)

    # Build Image with Experiments
    for experiment in experiments:
        exp_id = os.path.basename(experiment)
        if exp_id in library:
            config = library[exp_id]

            app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id)

            # Here add custom build routine, should be list of lines
            if "install" in config:
                commands = "\n".join(
                    ["RUN %s " % s for x in config['install']]).strip('\n')
                app = "%s%s\n" % (app, commands)

            # The final installation step, either from Github (url) or local folder
            if "local" in config:
                app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id)
                app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
                    app, exp_id)
            else:
                app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
                    app, config['github'])
            apps = "%s%s\n" % (apps, app)

        else:
            bot.warning('%s not in library, check spelling & punctuation.' %
                        exp_id)

    if apps == "\n":
        bot.error('No valid experiments found, cancelling recipe build.')
        sys.exit(1)

    template = sub_template(template, "{{experiments}}", apps)
    outfile = write_file(args.output, template)
    bot.log("Recipe written to %s" % outfile)
Ejemplo n.º 15
0
def generate_experiment_web(output_dir,experiment_folder=None,survey_folder=None,games_folder=None,
                            make_table=True,make_index=True,make_experiments=True,make_data=True,
                            make_surveys=True,make_games=True):
    '''get_experiment_table
    Generate a table with links to preview all experiments
    :param experiment_folder: folder with experiments inside
    :param survey_folder: folder with surveys inside
    :param games_folder: folder with games inside
    :param output_dir: output folder for experiment and table html, and battery files 
    :param make_table: generate table.html 
    :param make_index: generate d3 visualization index  
    :param make_experiments: generate experiment preview files (linked from table and index) 
    :param make_data: generate json/tsv data to download 
    :param make_surveys: generate static files for surveys repos 
    :param make_games: generate static files for games repos 
    '''
    repos=["experiments","battery"]
    if make_surveys == True:
        repos.append("surveys")

    if make_games == True:
        repos.append("games")     

    tmpdir = custom_battery_download(repos=repos)

    if not os.path.exists(output_dir):
        os.mkdir(output_dir)

    if experiment_folder == None:
        experiment_folder = "%s/experiments" %tmpdir
    experiments = get_experiments(experiment_folder,load=True, warning=False)
    experiment_tags = [x[0]["exp_id"] for x in experiments]
    battery_repo = "%s/battery" %(tmpdir)
    if survey_folder == None:
        survey_folder = "%s/surveys" %tmpdir
    if games_folder == None:
        games_folder = "%s/games" %tmpdir

    # If the user wants surveys and/or games, add them on to tasks
    tasks = experiments
    if make_surveys == True:
        surveys = get_experiments(survey_folder,load=True,warning=False,repo_type="surveys")
        survey_tags = [x[0]["exp_id"] for x in surveys]
        tasks = experiments + surveys

    if make_games == True:
        games = get_experiments(games_folder,load=True,warning=False,repo_type="games")
        games_tags = [x[0]["exp_id"] for x in games]
        tasks = tasks + games

    # Fields to keep for the table
    fields = ['preview','exp_id','template',
              'contributors','time',
              'cognitive_atlas_task_id']

    valid = pandas.DataFrame(columns=fields)
   
    # Make a table of experiment information
    for experiment in tasks:
        for field in experiment[0].keys():
            if field in fields:
                values = experiment[0][field]
                # Join lists with a comma
                if field == "reference":
                    if values != '':
                        values = '<a href="%s" target="_blank">%s</a>' %(values,values)
                if isinstance(values,list):
                    values = ",".join(values)
                valid.loc[experiment[0]["exp_id"],field] = values

        # Add a preview link
        valid.loc[experiment[0]["exp_id"],"preview"] = '<a href="%s.html" target="_blank">DEMO</a>' %(experiment[0]["exp_id"])
        
    # If the user wants to create the index page
    if make_index == True:
        output_index = os.path.abspath("%s/index.html" %output_dir)

        # For each experiment, we will prepare an interactive node for the site
        nodes = []
        for experiment in tasks:
            nodes.append('{"cluster": 1, "radius": "10", "color": colors[%s], "exp_id": "%s" }' %(choice([0,1,2]),experiment[0]["exp_id"]))

        # Generate index page
        index_template = get_template("%s/templates/expfactory_index.html" %get_installdir())        
        index_template = index_template.replace("[SUB_NODES_SUB]",",".join(nodes))
        index_template = index_template.replace("[SUB_TOTAL_SUB]",str(len(nodes)))
        filey = open(output_index,"wb")
        filey.writelines(index_template)
        filey.close()


    # Update entire static directory
    old_dirs = ["templates","static"]
    for folder in old_dirs:
        copy_to = os.path.abspath("%s/%s" %(output_dir,folder))
        copy_from = "%s/battery/%s" %(tmpdir,folder)
        if os.path.exists(copy_to):
            shutil.rmtree(copy_to)
        copy_directory(copy_from,copy_to)

    # Copy the favicon to web base
    shutil.copyfile("%s/battery/static/favicon.ico" %tmpdir,"%s/favicon.ico" %output_dir)

    # Clear old experiments
    experiment_dir = os.path.abspath("%s/static/experiments/" %output_dir)
    if os.path.exists(experiment_dir):
        shutil.rmtree(experiment_dir)

    # Clear old surveys, copy updated valid surveys into survey directory
    if make_surveys == True:
        survey_dir = os.path.abspath("%s/static/surveys/" %output_dir)
        if os.path.exists(survey_dir):
            shutil.rmtree(survey_dir)
        valid_surveys = ["%s/%s" %(survey_folder,x[0]["exp_id"]) for x in surveys]
        move_experiments(valid_surveys,battery_dest=output_dir,repo_type="surveys")

    # Clear old surveys, copy updated valid surveys into survey directory
    if make_games == True:
        games_dir = os.path.abspath("%s/static/games/" %output_dir)
        if os.path.exists(games_dir):
            shutil.rmtree(games_dir)
        valid_games = ["%s/%s" %(games_folder,x[0]["exp_id"]) for x in games]
        move_experiments(valid_games,battery_dest=output_dir,repo_type="games")

    # Copy updated valid experiments into our experiment directory
    valid_experiments = ["%s/%s" %(experiment_folder,x[0]["exp_id"]) for x in experiments]
    move_experiments(valid_experiments,battery_dest=output_dir)

    # If the user wants to make a table
    if make_table == True:

        table_template = get_template("%s/templates/table.html" %get_installdir())
        output_table = os.path.abspath("%s/table.html" %output_dir)

        # First prepare rendered table
        table = '<table id="fresh-table" class="table">\n<thead>\n'
        for field in fields:
            table = '%s<th data-field="%s" data-sortable="true">%s</th>' %(table,field,field)
        table = '%s\n</thead>\n<tbody>\n' %(table)

        for row in valid.iterrows():
            table = "%s<tr>\n" %(table)
            for field in row[1]:
                table = "%s<td>%s</td>\n" %(table,field)
            table = "%s</tr>\n" %(table)

        table = "%s</tbody></table>\n" %(table)

        # Write the new table
        table_template = table_template.replace("[[SUB_TABLE_SUB]]",table)
        filey = open("%s/table.html" %output_dir,"wb")
        filey.writelines(table_template)
        filey.close()

    if make_experiments == True:
        experiments_template = get_template("%s/templates/experiments_categories.html" %get_installdir())
        output_exp = os.path.abspath("%s/experiments.html" %output_dir)

        if "CIRCLE_BRANCH" in os.environ.keys():
            experiment_page = table_template    
        else:
            experiment_page = get_cognitiveatlas_hierarchy(experiment_tags=experiment_tags,get_html=True)

        # Write the new table
        filey = open(output_exp,"wb")
        filey.writelines(experiment_page)
        filey.close()
        
        # For each experiment, we will generate a demo page
        for experiment in experiments:
            demo_page = os.path.abspath("%s/%s.html" %(output_dir,experiment[0]["exp_id"]))
            exp_template = get_experiment_html(experiment,"%s/%s" %(experiment_folder,experiment[0]["exp_id"]))
            filey = open(demo_page,"wb")
            filey.writelines(exp_template)
            filey.close()

    # if the user wants to make surveys
    if make_surveys == True:
        for experiment in surveys:
            demo_page = os.path.abspath("%s/%s.html" %(output_dir,experiment[0]["exp_id"]))
            exp_template = get_experiment_html(experiment,"%s/%s" %(survey_folder,experiment[0]["exp_id"]))
            filey = open(demo_page,"wb")
            filey.writelines(exp_template)
            filey.close()


    # if the user wants to make surveys
    if make_games == True:
        for experiment in games:
            demo_page = os.path.abspath("%s/%s.html" %(output_dir,experiment[0]["exp_id"]))
            exp_template = get_experiment_html(experiment,"%s/%s" %(games_folder,experiment[0]["exp_id"]))
            filey = open(demo_page,"wb")
            filey.writelines(exp_template)
            filey.close()

    # If the user wants to make data
    if make_data == True:
        data_folder = os.path.abspath("%s/data" %output_dir)
        if not os.path.exists(data_folder):
            os.mkdir(data_folder)
        save_pretty_json("%s/expfactory-experiments.json" %(data_folder),json.loads(valid.to_json(orient="records")))
        valid.to_csv("%s/expfactory-experiments.tsv" %(data_folder),sep="\t",index=None)
        valid.to_pickle("%s/expfactory-experiments.pkl" %(data_folder))
Ejemplo n.º 16
0
def make_tree_from_triples(triples,
                           output_html=False,
                           meta_data=None,
                           delim="\t",
                           prune_tree=True):
    '''make_tree_from_triples
    generate a tree from a data structure with the following format
    :param triples: csv or pandas data frame 
        either a file or pandas data frame with this format

    ..note::
 
                             id             parent                          name
        0                     1               None                          BASE
        1     trm_4a3fd79d096be  trm_4a3fd79d0aec1           abductive reasoning
        2     trm_4a3fd79d096e3  trm_4a3fd79d09827              abstract analogy
        3     trm_4a3fd79d096f0  trm_4a3fd79d0a746            abstract knowledge
        4     trm_4a3fd79d096fc                  1               acoustic coding
    
        The index (0-4) is not used. The head/base node should have id 1, and no parent.
        All other nodes can be unique ids that you specify, with a name.

    :param delim: the delimiter use to separate values
    :param output_html: return html page instead of json data structure
    :param prune_tree: boolean do not include nodes that don't have task children default [True]
    :param meta_data [OPTIONAL]: dict 
          if defined, should be a dictionary of dictionaries, with

    ..note::

          key = the node id
          value = a dictionary of meta data. For example:

          {
            "trm_12345":{"defined_by":"squidward","score":1.2},
            "node_54321":{"date":"12/15/15"},
          }
    
    '''
    nodes = []

    if not isinstance(triples, pandas.DataFrame):
        triples = pandas.read_csv(relationship_table, sep="\t")

    # check for correct columns and circular references
    check_pandas_columns(df=triples, column_names=["id", "name", "parent"])
    find_circular_reference(triples)

    # Generate nodes
    unique_nodes = triples.id.unique().tolist()
    print("%s unique nodes found." % (len(unique_nodes)))
    for node in unique_nodes:
        parents = triples.parent[triples.id == node].tolist()
        name = triples.name[triples.id == node].unique().tolist()
        if len(name) > 1:
            raise ValueError("Error, node %s is defined with multiple names." %
                             node)
        meta = {}
        if meta_data:
            if node in meta_data:
                meta = meta_data[node]
        nodes.append(Node(node, parents, name[0], meta))

    # Generate json
    graph = get_json(nodes)

    if prune_tree == True:
        walk(graph, func=do_pruning)

    if output_html == True:
        # Generate the dynamic list - we will limit to three levels into the ontology
        html_snippet = ''
        # We will save a dictionary of base (experiment) nodes
        experiment_nodes = dict()
        # Save a list of nodes without concept_parents
        orphan_experiment_nodes = dict()
        for child in graph["children"]:
            # For each child, we will tag base nodes with parent ids
            parent_ids = [child["nid"]]
            # This first level cannot be a base node, so we don't check.
            # Update the experiment_nodes lookup with experiment nodes at this level
            exp_nodes = [
                x for x in child["children"] if re.search("node_", x["nid"])
            ]
            if len(exp_nodes) == 0:
                orphan_experiment_nodes = add_orphan_experiment_nodes(
                    orphan_experiment_nodes, child)
            experiment_nodes = add_experiment_nodes(experiment_nodes,
                                                    exp_nodes, parent_ids)
            # Do we have all base (experiment) nodes?
            if len(exp_nodes) == len(child["children"]):
                # Children do not get added to list
                html_snippet = '%s<a><li class="accord" id="accord_%s">%s</li></a>' % (
                    html_snippet, str(child["nid"]), str(child["name"]))
            else:
                html_snippet = '%s<li><a id="accord_%s" class="toggle accord">%s</a>' % (
                    html_snippet, str(child["nid"]), str(child["name"]))
                # Add non-children
                html_snippet = html_snippet + '<ul class="inner">'
                other_nodes = [
                    x for x in child["children"]
                    if not re.search("node_", x["nid"])
                ]
                for other_node in other_nodes:
                    sub_parent_ids = parent_ids[:]
                    sub_parent_ids.append(other_node["nid"])
                    # Update the experiment_nodes lookup with experiment nodes at this level
                    exp_nodes = [
                        x for x in other_node["children"]
                        if re.search("node_", x["nid"])
                    ]
                    if len(exp_nodes) == 0:
                        orphan_experiment_nodes = add_orphan_experiment_nodes(
                            orphan_experiment_nodes, other_node)
                    experiment_nodes = add_experiment_nodes(
                        experiment_nodes, exp_nodes, sub_parent_ids)
                    # Do we have all base (experiment) nodes?
                    if len(exp_nodes) == len(other_node["children"]):
                        html_snippet = '%s<a><li id="accord_%s" class="accord">%s</li></a>' % (
                            html_snippet, str(
                                other_node["nid"]), str(other_node["name"]))
                    else:
                        html_snippet = '%s<li><a id="accord_%s" class="toggle accord">%s</a>' % (
                            html_snippet, str(
                                other_node["nid"]), str(other_node["name"]))
                        html_snippet = html_snippet + '<ul class="inner">'
                        # Add non children
                        last_nodes = [
                            x for x in other_node["children"]
                            if not re.search("node_", x["nid"])
                        ]
                        for last_node in last_nodes:
                            last_parent_ids = sub_parent_ids[:]
                            last_parent_ids.append(last_node["nid"])
                            # One last final go to update experiment nodes
                            exp_nodes = [
                                x for x in last_node["children"]
                                if re.search("node_", x["nid"])
                            ]
                            if len(exp_nodes) == 0:
                                orphan_experiment_nodes = add_orphan_experiment_nodes(
                                    orphan_experiment_nodes, last_node)
                            experiment_nodes = add_experiment_nodes(
                                experiment_nodes, exp_nodes, last_parent_ids)
                            if len(exp_nodes) == len(last_node["children"]):
                                html_snippet = '%s<a><li id="accord_%s" class="accord">%s</li></a>' % (
                                    html_snippet, str(last_node["nid"]),
                                    str(last_node["name"]))
                            else:
                                html_snippet = '%s<li><a href="#" id="accord_%s" class="toggle accord">%s</a>' % (
                                    html_snippet, str(last_node["nid"]),
                                    str(last_node["name"]))
                                html_snippet = html_snippet + '<ul class="inner">'
                                base_nodes = [
                                    x for x in last_node["children"]
                                    if not re.search("node_", x["nid"])
                                ]  # Regardless of more layers, we generate links here for all remaining base_node links.
                                for base_node in base_nodes:
                                    html_snippet = '%s<a><li class="accord" id="accord_%s">%s</li></a>' % (
                                        html_snippet, str(base_node["nid"]),
                                        str(base_node["name"]))
                                    base_parent_ids = last_parent_ids[:]
                                    base_parent_ids.append(base_node["nid"])
                                    remaining_children = []
                                    remaining_parents = [base_node]
                                    while len(remaining_parents) > 0:
                                        current_node = remaining_parents.pop(0)
                                        remaining_children = remaining_children + [
                                            x for x in current_node["children"]
                                            if re.search("node_", x["nid"])
                                        ]
                                        remaining_parents = remaining_parents + [
                                            x for x in current_node["children"]
                                            if
                                            not re.search("node_", x["nid"])
                                        ]
                                    experiment_nodes = add_experiment_nodes(
                                        experiment_nodes, remaining_children,
                                        base_parent_ids)
                                html_snippet = "%s</ul></li>" % (html_snippet)
                        html_snippet = "%s</ul></li>" % (html_snippet)
                html_snippet = html_snippet + "</ul></li>"

        # Now we will generate html for each of the experiments, and save a lookup by concept id as we go
        concept_lookup = dict()
        html_experiments = ''
        orphan_experiment_nodes.update(experiment_nodes)
        for experiment_node, node in orphan_experiment_nodes.items():
            # If we have meta data, present each as a little paragraph.
            meta_snippet = ''
            if meta_data != None:
                if node["nid"] in meta_data:
                    meta_single = meta_data[node["nid"]]
                    for meta_tag, meta_value in meta_single.items():
                        if meta_tag != "experiment_variables":
                            if isinstance(meta_value, list):
                                meta_value = ",".join(meta_value)
                            meta_snippet = "%s<p><strong>%s</strong>: %s</p>" % (
                                meta_snippet, meta_tag, meta_value)
            html_experiments = '%s<div id="exp_%s" class="panel panel-default"><div class="alert-info" style="padding-left:10px;background-color:#F2DF49">%s</div><div class="panel-body"><p>%s</p><a href="https://expfactory.github.io/%s.html" target="_blank" class="btn btn-default">Preview Experiment</a></div></div>\n' % (
                html_experiments, node["name"], node["name"], meta_snippet,
                node["name"])
            for parent in node["parents"]:
                if str(parent) in concept_lookup:
                    holder = concept_lookup[str(parent)]
                    holder.append(experiment_node)
                    concept_lookup[str(parent)] = holder
                else:
                    concept_lookup[str(parent)] = [experiment_node]
            # If the node is an experiment and has only root parent, add to lookup
            if len(node["parents"]) == 1:
                if node["parents"][0] == "1":
                    concept_lookup[node["nid"]] = [experiment_node]
        # All experiments
        concept_lookup["all_experiments"] = orphan_experiment_nodes.keys()

        # Plug everything into the template
        template = get_template("%s/templates/experiments_categories.html" %
                                get_installdir())
        template = sub_template(template, "[[SUB_LOOKUP_SUB]]",
                                str(concept_lookup))
        template = sub_template(template, "[[SUB_EXPERIMENTS_SUB]]",
                                html_experiments)
        template = sub_template(template, "[[SUB_NAVIGATION_SUB]]",
                                html_snippet)
        graph = template
    return graph
Ejemplo n.º 17
0
def main(args,parser,subparser):

    folder = args.folder
    if folder is None:
        folder = os.getcwd()

    source = args.src[0]
    if source is None:
        bot.error('Please provide a Github https address to install.')
        sys.exit(1)

    # Is the experiment valid?
    cli = ExperimentValidator()
    valid = cli.validate(source, cleanup=False)
    exp_id = os.path.basename(source).replace('.git','')

    if valid is True:

        # Local Install
        if os.path.exists(source):
            config = load_experiment(source)
            source = os.path.abspath(source)
        else:
            config = load_experiment("%s/%s" %(cli.tmpdir,exp_id))
            source = "%s/%s" %(cli.tmpdir,exp_id)

        exp_id = config['exp_id']
        python_module = exp_id.replace('-','_').lower()
    else:
        bot.error('%s is not valid.' % exp_id)
        sys.exit(1)

    # Move static files to output folder
    dest = "%s/%s" %(folder,exp_id)

    bot.log("Installing %s to %s" %(exp_id, dest))
    
    # Building container
    in_container = False
    if os.environ.get('SINGULARITY_IMAGE') is not None:
        in_container = True

    # Running, live container
    elif os.environ.get('EXPFACTORY_CONTAINER') is not None:
        in_container = True

    if in_container is True:

        # if in container, we always force
        args.force = True

        bot.log("Preparing experiment routes...")
        template = get_template('experiments/template.py')
        template = sub_template(template, '{{ exp_id }}', exp_id)
        template = sub_template(template, '{{ exp_id_python }}', python_module)

        # 1. Python blueprint
        views = get_viewsdir(base=args.base)
        view_output = "%s/%s.py" %(views, python_module)
        save_template(view_output, template, base=views)
    
        # 2. append to __init__
        init = "%s/__init__.py" % views
        with open(init,'a') as filey:
            filey.writelines('from .%s import *\n' %python_module)

        # 3. Instructions
        if "instructions" in config:
            instruct = "%s/%s.help" %(views, python_module)
        with open(instruct,'w') as filey:
            filey.writelines(config['instructions'])

    if not os.path.exists(dest):
        os.system('mkdir -p %s' %dest)
    else:
        if args.force is False:
            bot.error('%s is not empty! Use --force to delete and re-create.' %folder)
            sys.exit(1) 

    # We don't need to copy if experiment already there
    if source != dest:
        os.system('cp -R %s/* %s' %(source, dest))