def validate_header(header, required_fields=None): """validate_header ensures that the first row contains the exp_id, var_name, var_value, and token. Capitalization isn't important, but ordering is. This criteria is very strict, but it's reasonable to require. Parameters ========== header: the header row, as a list required_fields: a list of required fields. We derive the required length from this list. Does not return, instead exits if malformed. Runs silently if OK. """ if required_fields is None: required_fields = ["exp_id", "var_name", "var_value", "token"] # The required length of the header based on required fields length = len(required_fields) # This is very strict, but no reason not to be header = _validate_row(header, required_length=length) header = [x.lower() for x in header] for idx in range(length): field = header[idx].lower().strip() if required_fields[idx] != field: bot.error("Malformed header field %s, exiting." % field) sys.exit(1)
def validate_header(header, required_fields=None): '''validate_header ensures that the first row contains the exp_id, var_name, var_value, and token. Capitalization isn't important, but ordering is. This criteria is very strict, but it's reasonable to require. Parameters ========== header: the header row, as a list required_fields: a list of required fields. We derive the required length from this list. Does not return, instead exits if malformed. Runs silently if OK. ''' if required_fields is None: required_fields = ['exp_id', 'var_name', 'var_value', 'token'] # The required length of the header based on required fields length = len(required_fields) # This is very strict, but no reason not to be header = _validate_row(header, required_length=length) header = [x.lower() for x in header] for idx in range(length): field = header[idx].lower().strip() if required_fields[idx] != field: bot.error('Malformed header field %s, exiting.' %field) sys.exit(1)
def clone(url, tmpdir=None): """clone a repository from Github""" if tmpdir is None: tmpdir = tempfile.mkdtemp() name = os.path.basename(url).replace(".git", "") dest = "%s/%s" % (tmpdir, name) return_code = os.system("git clone %s %s" % (url, dest)) if return_code == 0: return dest bot.error("Error cloning repo.") sys.exit(return_code)
def clone(url, tmpdir=None): '''clone a repository from Github''' if tmpdir is None: tmpdir = tempfile.mkdtemp() name = os.path.basename(url).replace('.git', '') dest = '%s/%s' % (tmpdir, name) return_code = os.system('git clone %s %s' % (url, dest)) if return_code == 0: return dest bot.error('Error cloning repo.') sys.exit(return_code)
def get_template(name, base=None): '''read in and return a template file ''' if base is None: base = get_templatedir() template_file = "%s/%s" % (base, name) if os.path.exists(template_file): with open(template_file, "r") as filey: template = "".join(filey.readlines()) return template bot.error("%s does not exist." % template_file)
def mkdir_p(path): """mkdir_p attempts to get the same functionality as mkdir -p :param path: the path to create. """ try: os.makedirs(path) except OSError as e: if e.errno == errno.EEXIST and os.path.isdir(path): pass else: bot.error("Error creating path %s, exiting." % path) sys.exit(1)
def get_page(self, url, name='Chrome'): '''get_page open a particular url, checking for Timeout ''' if self.browser is None: self.browser = self.get_browser(name) try: return self.browser.get(url) except TimeoutException: bot.error('Browser request timeout. Are you connected to the internet?') self.browser.close() sys.exit(1)
def load_experiment(folder, return_path=False): """load_experiment: reads in the config.json for a folder, returns None if not found. :param folder: full path to experiment folder :param return_path: if True, don't load the config.json, but return it """ fullpath = os.path.abspath(folder) config = "%s/config.json" % (fullpath) if not os.path.exists(config): bot.error("config.json could not be found in %s" % (folder)) config = None if return_path is False and config is not None: config = read_json(config) return config
def validate(self, url): """ takes in a Github repository for validation of preview and runtime (and possibly tests passing? """ # Preview must provide the live URL of the repository if not url.startswith("http") or not "github" in url: bot.error("Test of preview must be given a Github repostitory.") return False if not self._validate_preview(url): return False return True
def _validate_markdown(self, expfile): """ensure that fields are present in markdown file""" try: import yaml except: bot.error( "Python yaml is required for testing yml/markdown files.") sys.exit(1) self.metadata = {} uid = os.path.basename(expfile).strip(".md") if os.path.exists(expfile): with open(expfile, "r") as stream: docs = yaml.load_all(stream) for doc in docs: if isinstance(doc, dict): for k, v in doc.items(): print("%s: %s" % (k, v)) self.metadata[k] = v self.metadata["uid"] = uid fields = [ "github", "preview", "name", "layout", "tags", "uid", "maintainer", ] # Tests for all fields for field in fields: if field not in self.metadata: return False if self.metadata[field] in ["", None]: return False if "github" not in self.metadata["github"]: return notvalid("%s: not a valid github repository" % name) if not isinstance(self.metadata["tags"], list): return notvalid("%s: tags must be a list" % name) if not re.search("(\w+://)(.+@)*([\w\d\.]+)(:[\d]+){0,1}/*(.*)", self.metadata["github"]): return notvalid("%s is not a valid URL." % (self.metadata["github"])) return True
def validate(self, url): ''' takes in a Github repository for validation of preview and runtime (and possibly tests passing? ''' # Preview must provide the live URL of the repository if not url.startswith('http') or not 'github' in url: bot.error('Test of preview must be given a Github repostitory.') return False if not self._validate_preview(url): return False return True
def load_experiment(folder, return_path=False): '''load_experiment: reads in the config.json for a folder, returns None if not found. :param folder: full path to experiment folder :param return_path: if True, don't load the config.json, but return it ''' fullpath = os.path.abspath(folder) config = "%s/config.json" %(fullpath) if not os.path.exists(config): bot.error("config.json could not be found in %s" %(folder)) config = None if return_path is False and config is not None: config = read_json(config) return config
def validate(self, url): ''' takes in a Github repository for validation of preview and runtime (and possibly tests passing? ''' # Preview must provide the live URL of the repository if not url.startswith('http') or not 'github' in url: bot.error('Test of preview must be given a Github repostitory.') return False if not self._validate_preview(url): return False return True
def copy_directory(src, dest, force=False): ''' Copy an entire directory recursively ''' if os.path.exists(dest) and force is True: shutil.rmtree(dest) try: shutil.copytree(src, dest) except OSError as e: # If the error was caused because the source wasn't a directory if e.errno == errno.ENOTDIR: shutil.copy(src, dest) else: bot.error('Directory not copied. Error: %s' % e) sys.exit(1)
def get_template(name, base=None): """read in and return a template file """ # If the file doesn't exist, assume relative to base template_file = name if not os.path.exists(template_file): if base is None: base = get_templatedir() template_file = "%s/%s" % (base, name) # Then try again, if it still doesn't exist, bad name if os.path.exists(template_file): with open(template_file, "r") as filey: template = "".join(filey.readlines()) return template bot.error("%s does not exist." % template_file)
def main(args,parser,subparser=None): '''this is the main entrypoint for a container based web server, with most of the variables coming from the environment. See the Dockerfile template for how this function is executed. ''' # First priority to args.base base = args.base if base is None: base = os.environ.get('EXPFACTORY_BASE') # Does the base folder exist? if base is None: bot.error("You must set a base of experiments with --base" % base) sys.exit(1) if not os.path.exists(base): bot.error("Base folder %s does not exist." % base) sys.exit(1) # Export environment variables for the client experiments = args.experiments if experiments is None: experiments = " ".join(glob("%s/*" % base)) os.environ['EXPFACTORY_EXPERIMENTS'] = experiments # If defined and file exists, set runtime variables if args.vars is not None: if os.path.exists(args.vars): os.environ['EXPFACTORY_RUNTIME_VARS'] = args.vars os.environ['EXPFACTORY_RUNTIME_DELIM'] = args.delim else: bot.warning('Variables file %s not found.' %args.vars) subid = os.environ.get('EXPFACTORY_STUDY_ID') if args.subid is not None: subid = args.subid os.environ['EXPFACTORY_SUBID'] = subid os.environ['EXPFACTORY_RANDOM'] = str(args.disable_randomize) os.environ['EXPFACTORY_BASE'] = base from expfactory.server import start start(port=5000)
def main(args, parser, subparser=None): """this is the main entrypoint for a container based web server, with most of the variables coming from the environment. See the Dockerfile template for how this function is executed. """ # First priority to args.base base = args.base if base is None: base = os.environ.get("EXPFACTORY_BASE") # Does the base folder exist? if base is None: bot.error("You must set a base of experiments with --base" % base) sys.exit(1) if not os.path.exists(base): bot.error("Base folder %s does not exist." % base) sys.exit(1) # Export environment variables for the client experiments = args.experiments if experiments is None: experiments = " ".join(glob("%s/*" % base)) os.environ["EXPFACTORY_EXPERIMENTS"] = experiments # If defined and file exists, set runtime variables if args.vars is not None: if os.path.exists(args.vars): os.environ["EXPFACTORY_RUNTIME_VARS"] = args.vars os.environ["EXPFACTORY_RUNTIME_DELIM"] = args.delim else: bot.warning("Variables file %s not found." % args.vars) subid = os.environ.get("EXPFACTORY_STUDY_ID") if args.subid is not None: subid = args.subid os.environ["EXPFACTORY_SUBID"] = subid os.environ["EXPFACTORY_RANDOM"] = str(args.disable_randomize) os.environ["EXPFACTORY_BASE"] = base from expfactory.server import start start(port=5000)
def _validate_markdown(self, expfile): '''ensure that fields are present in markdown file''' try: import yaml except: bot.error( 'Python yaml is required for testing yml/markdown files.') sys.exit(1) self.metadata = {} uid = os.path.basename(expfile).strip('.md') if os.path.exists(expfile): with open(expfile, "r") as stream: docs = yaml.load_all(stream) for doc in docs: if isinstance(doc, dict): for k, v in doc.items(): print('%s: %s' % (k, v)) self.metadata[k] = v self.metadata['uid'] = uid fields = [ 'github', 'preview', 'name', 'layout', 'tags', 'uid', 'maintainer' ] # Tests for all fields for field in fields: if field not in self.metadata: return False if self.metadata[field] in ['', None]: return False if 'github' not in self.metadata['github']: return notvalid('%s: not a valid github repository' % name) if not isinstance(self.metadata['tags'], list): return notvalid('%s: tags must be a list' % name) if not re.search("(\w+://)(.+@)*([\w\d\.]+)(:[\d]+){0,1}/*(.*)", self.metadata['github']): return notvalid('%s is not a valid URL.' % (self.metadata['github'])) return True
def main(): parser = get_parser() try: args = parser.parse_args() except: sys.exit(0) folder = args.folder if folder is None: folder = os.getcwd() folder = os.path.abspath(folder) survey = "%s/survey.tsv" % folder if not os.path.exists(survey): bot.error("Cannot find %s, required to generate survey." % survey) sys.exit(1) config = load_experiment(folder) html, validation = generate_survey(config=config, survey_file=survey, form_action=args.action) output = args.output if output is None: output = folder output_index = "%s/index.html" % folder if os.path.exists(output_index) and args.force is False: bot.error("%s already exists, use --force to overwrite." % output_index) sys.exit(1) bot.info("Writing output files to %s" % output_index) template = get_template('survey/index.html') template = sub_template(template, "{{html}}", html) template = sub_template(template, "{{exp_id}}", config['exp_id']) template = sub_template(template, "{{validation}}", validation) write_file(output_index, template)
def getenv(variable_key, default=None, required=False, silent=True): """getenv will attempt to get an environment variable. If the variable is not found, None is returned. :param variable_key: the variable name :param required: exit with error if not found :param silent: Do not print debugging information for variable """ variable = os.environ.get(variable_key, default) if variable is None and required: bot.error("Cannot find environment variable %s, exiting." % variable_key) sys.exit(1) if not silent: if variable is not None: bot.verbose2("%s found as %s" % (variable_key, variable)) else: bot.verbose2("%s not defined (None)" % variable_key) return variable
def _validate_preview(self, url): bot.test("Experiment url: %s" % url) org, repo = url.split("/")[-2:] if repo.endswith(".git"): repo = repo.replace(".git", "") github_pages = "https://%s.github.io/%s" % (org, repo) bot.test("Github Pages url: %s" % github_pages) response = requests.get(github_pages) if response.status_code == 404: bot.error("""Preview not found at %s. You must publish a static preview from the master branch of your repository to add to the library.""" % github_pages) return False index = response.text tmpdir = tempfile.mkdtemp() repo_master = clone(url, tmpdir) contenders = glob("%s/*" % repo_master) license = False found = False for test in contenders: if os.path.isdir(test): continue print("...%s" % test) if "LICENSE" in os.path.basename(test): license = True if os.path.basename(test) == "index.html": bot.test("Found index file in repository.") found = True break if license is False: bot.warning( "LICENSE file not found. This will be required for future experiments!" ) self._print_valid(found) return found
def _validate_preview(self, url): bot.test('Experiment url: %s' %url) org,repo = url.split('/')[-2:] if repo.endswith('.git'): repo = repo.replace('.git','') github_pages = "https://%s.github.io/%s" %(org,repo) bot.test('Github Pages url: %s' %github_pages) response = requests.get(github_pages) if response.status_code == 404: bot.error('''Preview not found at %s. You must publish a static preview from the master branch of your repository to add to the library.''' % github_pages) return False index = response.text tmpdir = tempfile.mkdtemp() repo_master = clone(url, tmpdir) contenders = glob('%s/*' %repo_master) license = False found = False for test in contenders: if os.path.isdir(test): continue print('...%s' %test) if "LICENSE" in os.path.basename(test): license = True if os.path.basename(test) == "index.html": bot.test('Found index file in repository.') found = True break if license is False: bot.warning("LICENSE file not found. This will be required for future experiments!") self._print_valid(found) return found
def _validate_markdown(self, expfile): '''ensure that fields are present in markdown file''' try: import yaml except: bot.error('Python yaml is required for testing yml/markdown files.') sys.exit(1) self.metadata = {} uid = os.path.basename(expfile).strip('.md') if os.path.exists(expfile): with open(expfile, "r") as stream: docs = yaml.load_all(stream) for doc in docs: if isinstance(doc,dict): for k,v in doc.items(): print('%s: %s' %(k,v)) self.metadata[k] = v self.metadata['uid'] = uid fields = ['github', 'preview', 'name', 'layout', 'tags', 'uid', 'maintainer'] # Tests for all fields for field in fields: if field not in self.metadata: return False if self.metadata[field] in ['',None]: return False if 'github' not in self.metadata['github']: return notvalid('%s: not a valid github repository' % name) if not isinstance(self.metadata['tags'],list): return notvalid('%s: tags must be a list' % name) if not re.search("(\w+://)(.+@)*([\w\d\.]+)(:[\d]+){0,1}/*(.*)", self.metadata['github']): return notvalid('%s is not a valid URL.' %(self.metadata['github'])) return True
def main(): parser = get_parser() try: args = parser.parse_args() except: sys.exit(0) print('Recruiting %s robot!' % args.robot) folders = args.folders if len(folders) == 0: folders = [os.getcwd()] # The drivers must be on path here = os.path.abspath(os.path.dirname(__file__)) os.environ['PATH'] = "%s/drivers:%s" % (here, os.environ['PATH']) # Load the robot! if args.robot == 'jspsych': from drivers.jspsych import JsPsychRobot as Robot elif args.robot == 'survey': from drivers.survey import SurveyRobot as Robot robot = Robot(browser=args.browser, port=args.port, headless=args.headless) for folder in folders: folder = os.path.abspath(folder) if not os.path.exists(folder): bot.error("Cannot find %s, check that path exists." % folder) else: print('[folder] %s' % folder) robot.validate(folder) # Clean up shop! robot.stop()
def main(args, parser, subparser=None): '''this is the main entrypoint for a container based web server, with most of the variables coming from the environment. See the Dockerfile template for how this function is executed. ''' # First priority to args.base base = args.base if base is None: base = os.environ.get('EXPFACTORY_BASE') # Does the base folder exist? if base is None: bot.error("You must set a base of experiments with --base" % base) sys.exit(1) if not os.path.exists(base): bot.error("Base folder %s does not exist." % base) sys.exit(1) # Export environment variables for the client experiments = args.experiments if experiments is None: experiments = " ".join(glob("%s/*" % base)) os.environ['EXPFACTORY_EXPERIMENTS'] = experiments subid = os.environ.get('EXPFACTORY_STUDY_ID') if args.subid is not None: subid = args.subid os.environ['EXPFACTORY_SUBID'] = subid os.environ['EXPFACTORY_RANDOM'] = str(args.disable_randomize) os.environ['EXPFACTORY_BASE'] = base from expfactory.server import start start(port=5000)
def main(args,parser,subparser): folder = args.folder if folder is None: folder = os.getcwd() source = args.src[0] if source is None: bot.error('Please provide a Github https address to install.') sys.exit(1) # Is the experiment valid? cli = ExperimentValidator() valid = cli.validate(source, cleanup=False) exp_id = os.path.basename(source).replace('.git','') if valid is True: # Local Install if os.path.exists(source): config = load_experiment(source) source = os.path.abspath(source) else: config = load_experiment("%s/%s" %(cli.tmpdir,exp_id)) source = "%s/%s" %(cli.tmpdir,exp_id) exp_id = config['exp_id'] python_module = exp_id.replace('-','_').lower() else: bot.error('%s is not valid.' % exp_id) sys.exit(1) # Move static files to output folder dest = "%s/%s" %(folder,exp_id) bot.log("Installing %s to %s" %(exp_id, dest)) # Building container in_container = False if os.environ.get('SINGULARITY_IMAGE') is not None: in_container = True # Running, live container elif os.environ.get('EXPFACTORY_CONTAINER') is not None: in_container = True if in_container is True: # if in container, we always force args.force = True bot.log("Preparing experiment routes...") template = get_template('experiments/template.py') template = sub_template(template, '{{ exp_id }}', exp_id) template = sub_template(template, '{{ exp_id_python }}', python_module) # 1. Python blueprint views = get_viewsdir(base=args.base) view_output = "%s/%s.py" %(views, python_module) save_template(view_output, template, base=views) # 2. append to __init__ init = "%s/__init__.py" % views with open(init,'a') as filey: filey.writelines('from .%s import *\n' %python_module) # 3. Instructions if "instructions" in config: instruct = "%s/%s.help" %(views, python_module) with open(instruct,'w') as filey: filey.writelines(config['instructions']) if not os.path.exists(dest): os.system('mkdir -p %s' %dest) else: if args.force is False: bot.error('%s is not empty! Use --force to delete and re-create.' %folder) sys.exit(1) # We don't need to copy if experiment already there if source != dest: os.system('cp -R %s/* %s' %(source, dest))
def main(args, parser, subparser): # List of experiments is required template = get_template('build/docker/Dockerfile.template') # For now, only one database provided database = args.database studyid = args.studyid experiments = args.experiments template = sub_template(template, "{{studyid}}", studyid) template = sub_template(template, "{{database}}", database) library = get_library(key='name') apps = "\n" # Add local experiments to library, first preference local_installs = 0 for experiment in experiments: if os.path.exists(experiment): bot.info('local experiment %s found, validating...' % experiment) # Is the experiment valid? cli = ExperimentValidator() valid = cli.validate(experiment, cleanup=False) if valid is True: local_installs += 1 config = load_experiment(experiment) exp_id = config['exp_id'] # If we aren't building in the experiment directory, we need to copy there output_dir = "%s/%s" % (os.path.abspath( os.path.dirname(args.output)), exp_id) experiment_dir = os.path.abspath(experiment) if output_dir != experiment_dir: copy_directory(experiment_dir, output_dir) config['local'] = os.path.abspath(experiment) library[exp_id] = config # Warn the user that local installs are not reproducible (from recipe) if local_installs > 0: bot.warning( "%s local installs detected: build is not reproducible without experiment folders" % local_installs) # Build Image with Experiments for experiment in experiments: exp_id = os.path.basename(experiment) if exp_id in library: config = library[exp_id] app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id) # Here add custom build routine, should be list of lines if "install" in config: commands = "\n".join( ["RUN %s " % s for x in config['install']]).strip('\n') app = "%s%s\n" % (app, commands) # The final installation step, either from Github (url) or local folder if "local" in config: app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id) app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % ( app, exp_id) else: app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % ( app, config['github']) apps = "%s%s\n" % (apps, app) else: bot.warning('%s not in library, check spelling & punctuation.' % exp_id) if apps == "\n": bot.error('No valid experiments found, cancelling recipe build.') sys.exit(1) template = sub_template(template, "{{experiments}}", apps) outfile = write_file(args.output, template) bot.log("Recipe written to %s" % outfile)
def main(args,parser,subparser): template = 'build/docker/Dockerfile.template' # Full path to template is required if provided via input if args.input is not None: template = args.input template = get_template(template) # For now, only one database provided database = args.database studyid = args.studyid experiments = args.experiments branch = "-b %s" %os.environ.get('EXPFACTORY_BRANCH','master') headless = "false" if args.headless is True: headless = "true" template = sub_template(template,"{{studyid}}",studyid) template = sub_template(template,"{{database}}",database) template = sub_template(template,"{{headless}}",headless) template = sub_template(template,"{{branch}}",branch) if args.headless is True: bot.info("Headless build detected, you will need to generate tokens for application entry with expfactory users --new") library = get_library(key='name') apps = "\n" # Add local experiments to library, first preference local_installs = 0 for experiment in experiments: if os.path.exists(experiment): bot.info('local experiment %s found, validating...' %experiment) # Is the experiment valid? cli = ExperimentValidator() valid = cli.validate(experiment, cleanup=False) if valid is True: local_installs +=1 config = load_experiment(experiment) exp_id = config['exp_id'] # If we aren't building in the experiment directory, we need to copy there output_dir = "%s/%s" %(os.path.abspath(os.path.dirname(args.output)), exp_id) experiment_dir = os.path.abspath(experiment) if output_dir != experiment_dir: copy_directory(experiment_dir, output_dir) config['local'] = os.path.abspath(experiment) library[exp_id] = config # Warn the user that local installs are not reproducible (from recipe) if local_installs > 0: bot.warning("%s local installs detected: build is not reproducible without experiment folders" %local_installs) # Build Image with Experiments for experiment in experiments: exp_id = os.path.basename(experiment) if exp_id in library: config = library[exp_id] app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" %(exp_id, exp_id) # Here add custom build routine, should be list of lines if "install" in config: commands = "\n".join(["RUN %s "%s for x in config['install']]).strip('\n') app = "%s%s\n" %(app, commands) # The final installation step, either from Github (url) or local folder if "local" in config: app = "%sADD %s /scif/apps/%s\n" %(app, exp_id, exp_id) app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, exp_id) else: app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, config['github']) apps = "%s%s\n" %(apps,app) else: bot.warning('%s not in library, check spelling & punctuation.' %exp_id) if apps == "\n": bot.error('No valid experiments found, cancelling recipe build.') sys.exit(1) template = sub_template(template,"{{experiments}}",apps) outfile = write_file(args.output,template) bot.log("Recipe written to %s" %outfile)
def main(args, parser, subparser): folder = args.folder if folder is None: folder = os.getcwd() source = args.src[0] if source is None: bot.error('Please provide a Github https address to install.') sys.exit(1) # Is the experiment valid? cli = ExperimentValidator() valid = cli.validate(source, cleanup=False) exp_id = os.path.basename(source).replace('.git', '') if valid is True: # Local Install if os.path.exists(source): config = load_experiment(source) source = os.path.abspath(source) else: config = load_experiment("%s/%s" % (cli.tmpdir, exp_id)) source = "%s/%s" % (cli.tmpdir, exp_id) exp_id = config['exp_id'] python_module = exp_id.replace('-', '_').lower() else: bot.error('%s is not valid.' % exp_id) sys.exit(1) # Move static files to output folder dest = "%s/%s" % (folder, exp_id) bot.log("Installing %s to %s" % (exp_id, dest)) # Building container in_container = False if os.environ.get('SINGULARITY_IMAGE') is not None: in_container = True # Running, live container elif os.environ.get('EXPFACTORY_CONTAINER') is not None: in_container = True if in_container is True: # if in container, we always force args.force = True bot.log("Preparing experiment routes...") template = get_template('experiments/template.py') template = sub_template(template, '{{ exp_id }}', exp_id) template = sub_template(template, '{{ exp_id_python }}', python_module) # 1. Python blueprint views = get_viewsdir(base=args.base) view_output = "%s/%s.py" % (views, python_module) save_template(view_output, template, base=views) # 2. append to __init__ init = "%s/__init__.py" % views with open(init, 'a') as filey: filey.writelines('from .%s import *\n' % python_module) # 3. Instructions if "instructions" in config: instruct = "%s/%s.help" % (views, python_module) with open(instruct, 'w') as filey: filey.writelines(config['instructions']) if not os.path.exists(dest): os.system('mkdir -p %s' % dest) else: if args.force is False: bot.error('%s is not empty! Use --force to delete and re-create.' % folder) sys.exit(1) # We don't need to copy if experiment already there if source != dest: os.system('cp -R %s/* %s' % (source, dest))
def main(args, parser, subparser): template = "build/docker/Dockerfile.template" # Full path to template is required if provided via input if args.input is not None: template = args.input template = get_template(template) # For now, only one database provided database = args.database studyid = args.studyid experiments = args.experiments branch = "-b %s" % os.environ.get("EXPFACTORY_BRANCH", "master") headless = "false" if args.headless is True: headless = "true" template = sub_template(template, "{{studyid}}", studyid) template = sub_template(template, "{{database}}", database) template = sub_template(template, "{{headless}}", headless) template = sub_template(template, "{{branch}}", branch) if args.headless is True: bot.info( "Headless build detected, you will need to generate tokens for application entry with expfactory users --new" ) library = get_library(key="name") apps = "\n" # Add local experiments to library, first preference local_installs = 0 for experiment in experiments: if os.path.exists(experiment): bot.info("local experiment %s found, validating..." % experiment) # Is the experiment valid? cli = ExperimentValidator() valid = cli.validate(experiment, cleanup=False) if valid is True: local_installs += 1 config = load_experiment(experiment) exp_id = config["exp_id"] # If we aren't building in the experiment directory, we need to copy there output_dir = "%s/%s" % ( os.path.abspath(os.path.dirname(args.output)), exp_id, ) experiment_dir = os.path.abspath(experiment) if output_dir != experiment_dir: copy_directory(experiment_dir, output_dir) config["local"] = os.path.abspath(experiment) library[exp_id] = config # Warn the user that local installs are not reproducible (from recipe) if local_installs > 0: bot.warning( "%s local installs detected: build is not reproducible without experiment folders" % local_installs ) # Build Image with Experiments for experiment in experiments: exp_id = os.path.basename(experiment) if exp_id in library: config = library[exp_id] app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id) # Here add custom build routine, should be list of lines if "install" in config: commands = "\n".join(["RUN %s " % s for x in config["install"]]).strip( "\n" ) app = "%s%s\n" % (app, commands) # The final installation step, either from Github (url) or local folder if "local" in config: app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id) app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % ( app, exp_id, ) else: app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % ( app, config["github"], ) apps = "%s%s\n" % (apps, app) else: bot.warning("%s not in library, check spelling & punctuation." % exp_id) if apps == "\n": bot.error("No valid experiments found, cancelling recipe build.") sys.exit(1) template = sub_template(template, "{{experiments}}", apps) outfile = write_file(args.output, template) bot.log("Recipe written to %s" % outfile)