def init_db(self): '''initialize the database, with the default database path or custom of the format sqlite:////scif/data/expfactory.db ''' # Database Setup, use default if uri not provided if self.database == 'sqlite': db_path = os.path.join(EXPFACTORY_DATA, '%s.db' % EXPFACTORY_SUBID) self.database = 'sqlite:///%s' % db_path bot.info("Database located at %s" % self.database) self.engine = create_engine(self.database, convert_unicode=True) self.session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=self.engine)) Base.query = self.session.query_property() # import all modules here that might define models so that # they will be registered properly on the metadata. Otherwise # you will have to import them first before calling init_db() import expfactory.database.models Base.metadata.create_all(bind=self.engine) self.Base = Base
def main(): parser = get_parser() subparsers = get_subparsers(parser) try: args = parser.parse_args() except: sys.exit(0) # Does the use want to install? command = args.command # Options that shouldn't produce output if command in ['users']: os.environ['MESSAGELEVEL'] = "0" if args.database is not None: os.environ['EXPFACTORY_DATABASE'] = args.database from expfactory.logger import bot from expfactory.version import __version__ bot.info("Expfactory Version: %s" % __version__) if command == "install": from .install import main elif command == "list": from .list import main elif command == "logs": from .logs import main elif command == "users": from .users import main elif command == "build": from .build import main # No argument supplied else: # A base exists for experiments base = os.environ.get('EXPFACTORY_BASE') if args.base is not None or base is not None: from .main import main command = "main" else: command = "list" from .list import main # Pass on to the correct parser if command is not None: # Main doesn't have a subparser subparser = None if command != "main": subparser = subparsers[command] main(args=args, parser=parser, subparser=subparser)
def main(args, parser, subparser): from expfactory.server import app header = 'DATABASE\tTOKEN' # The user wants to list active subjects if args.list is True: users = app.list_users() # returns id\ttoken sys.exit(0) # The user wants to add new subjects number = args.new if number is not None: print(header) for i in range(number): user = app.generate_user() app.print_user(user) sys.exit(0) # The user wants to manage user token action = None if args.revoke is not None: subid = clean(args.revoke) func = app.revoke_token action = "Revoking" elif args.refresh is not None: subid = clean(args.refresh) func = app.refresh_token action = "Refreshing" elif args.restart is not None: subid = clean(args.restart) func = app.restart_user action = "Restarting" elif args.finish is not None: subid = clean(args.finish) action = "Finishing" func = app.finish_user # Perform the action if action is not None: bot.info('%s %s' %(action, subid)) result = func(subid=subid) if result is not None: print("[%s] %s --> %s" %(action.lower(), subid, result)) else: print("[%s] not successful. See logs for details." %(action.lower())) print("Commands may only possible for [active] status.") sys.exit(0) print('See expfactory users --help for usage')
def main(args, parser, subparser): response = requests.get(EXPFACTORY_LIBRARY) if response.status_code == 200: library = response.json() bot.info("Experiments") rows = [] for experiment in library: rows.append([experiment["name"], experiment["github"]]) bot.table(rows)
def main(args,parser,subparser): response = requests.get(EXPFACTORY_LIBRARY) if response.status_code == 200: library = response.json() bot.info("Experiments") rows = [] for experiment in library: rows.append([ experiment['name'], experiment['github'] ]) bot.table(rows)
def get_experiments(base, load=False): """ get_experiments will return loaded json for all valid experiments from an experiment folder :param base: full path to the base folder with experiments inside :param load: if True, returns a list of loaded config.json objects. If False (default) returns the paths to the experiments """ experiments = find_directories(base) valid_experiments = [e for e in experiments if validate(e, cleanup=False)] bot.info("Found %s valid experiments" % (len(valid_experiments))) if load is True: valid_experiments = load_experiments(valid_experiments) # TODO at some point in this workflow we would want to grab instructions from help # and variables from labels, environment, etc. return valid_experiments
def get_experiments(base, load=False): ''' get_experiments will return loaded json for all valid experiments from an experiment folder :param base: full path to the base folder with experiments inside :param load: if True, returns a list of loaded config.json objects. If False (default) returns the paths to the experiments ''' experiments = find_directories(base) valid_experiments = [e for e in experiments if validate(e,cleanup=False)] bot.info("Found %s valid experiments" %(len(valid_experiments))) if load is True: valid_experiments = load_experiments(valid_experiments) #TODO at some point in this workflow we would want to grab instructions from help # and variables from labels, environment, etc. return valid_experiments
def main(): parser = get_parser() try: args = parser.parse_args() except: sys.exit(0) folder = args.folder if folder is None: folder = os.getcwd() folder = os.path.abspath(folder) survey = "%s/survey.tsv" % folder if not os.path.exists(survey): bot.error("Cannot find %s, required to generate survey." % survey) sys.exit(1) config = load_experiment(folder) html, validation = generate_survey(config=config, survey_file=survey, form_action=args.action) output = args.output if output is None: output = folder output_index = "%s/index.html" % folder if os.path.exists(output_index) and args.force is False: bot.error("%s already exists, use --force to overwrite." % output_index) sys.exit(1) bot.info("Writing output files to %s" % output_index) template = get_template('survey/index.html') template = sub_template(template, "{{html}}", html) template = sub_template(template, "{{exp_id}}", config['exp_id']) template = sub_template(template, "{{validation}}", validation) write_file(output_index, template)
def initdb(self): '''initdb will check for writability of the data folder, meaning that it is bound to the local machine. If the folder isn't bound, expfactory runs in demo mode (not saving data) ''' self.database = EXPFACTORY_DATABASE bot.info("DATABASE: %s" %self.database) # Supported database options valid = ('sqlite', 'postgres', 'mysql', 'filesystem') if not self.database.startswith(valid): bot.warning('%s is not yet a supported type, saving to filesystem.' % self.database) self.database = 'filesystem' # Add functions specific to database type self.init_db() # uses url in self.database bot.log("Data base: %s" % self.database)
def save_data(self, session, exp_id, content): '''save data will obtain the current subid from the session, and save it depending on the database type.''' from expfactory.database.models import (Participant, Result) subid = session.get('subid') bot.info('Saving data for subid %s' % subid) # We only attempt save if there is a subject id, set at start if subid is not None: p = Participant.query.filter( Participant.id == subid).first() # better query here # Preference is to save data under 'data', otherwise do all of it if "data" in content: content = content['data'] if isinstance(content, dict): content = json.dumps(content) result = Result(data=content, exp_id=exp_id, participant_id=p.id) # check if changes from str/int self.session.add(result) p.results.append(result) self.session.commit() bot.info("Participant: %s" % p) bot.info("Result: %s" % result)
def save_data(self,session, exp_id, content): '''save data will obtain the current subid from the session, and save it depending on the database type.''' from expfactory.database.models import ( Participant, Result ) subid = session.get('subid') bot.info('Saving data for subid %s' % subid) # We only attempt save if there is a subject id, set at start if subid is not None: p = Participant.query.filter(Participant.id == subid).first() # better query here # Preference is to save data under 'data', otherwise do all of it if "data" in content: content = content['data'] if isinstance(content,dict): content = json.dumps(content) result = Result(data=content, exp_id=exp_id, participant_id=p.id) # check if changes from str/int self.session.add(result) p.results.append(result) self.session.commit() bot.info("Participant: %s" %p) bot.info("Result: %s" %result)
def main(args, parser, subparser): template = "build/docker/Dockerfile.template" # Full path to template is required if provided via input if args.input is not None: template = args.input template = get_template(template) # For now, only one database provided database = args.database studyid = args.studyid experiments = args.experiments branch = "-b %s" % os.environ.get("EXPFACTORY_BRANCH", "master") headless = "false" if args.headless is True: headless = "true" template = sub_template(template, "{{studyid}}", studyid) template = sub_template(template, "{{database}}", database) template = sub_template(template, "{{headless}}", headless) template = sub_template(template, "{{branch}}", branch) if args.headless is True: bot.info( "Headless build detected, you will need to generate tokens for application entry with expfactory users --new" ) library = get_library(key="name") apps = "\n" # Add local experiments to library, first preference local_installs = 0 for experiment in experiments: if os.path.exists(experiment): bot.info("local experiment %s found, validating..." % experiment) # Is the experiment valid? cli = ExperimentValidator() valid = cli.validate(experiment, cleanup=False) if valid is True: local_installs += 1 config = load_experiment(experiment) exp_id = config["exp_id"] # If we aren't building in the experiment directory, we need to copy there output_dir = "%s/%s" % ( os.path.abspath(os.path.dirname(args.output)), exp_id, ) experiment_dir = os.path.abspath(experiment) if output_dir != experiment_dir: copy_directory(experiment_dir, output_dir) config["local"] = os.path.abspath(experiment) library[exp_id] = config # Warn the user that local installs are not reproducible (from recipe) if local_installs > 0: bot.warning( "%s local installs detected: build is not reproducible without experiment folders" % local_installs ) # Build Image with Experiments for experiment in experiments: exp_id = os.path.basename(experiment) if exp_id in library: config = library[exp_id] app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id) # Here add custom build routine, should be list of lines if "install" in config: commands = "\n".join(["RUN %s " % s for x in config["install"]]).strip( "\n" ) app = "%s%s\n" % (app, commands) # The final installation step, either from Github (url) or local folder if "local" in config: app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id) app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % ( app, exp_id, ) else: app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % ( app, config["github"], ) apps = "%s%s\n" % (apps, app) else: bot.warning("%s not in library, check spelling & punctuation." % exp_id) if apps == "\n": bot.error("No valid experiments found, cancelling recipe build.") sys.exit(1) template = sub_template(template, "{{experiments}}", apps) outfile = write_file(args.output, template) bot.log("Recipe written to %s" % outfile)
def main(args, parser, subparser): # List of experiments is required template = get_template('build/docker/Dockerfile.template') # For now, only one database provided database = args.database studyid = args.studyid experiments = args.experiments template = sub_template(template, "{{studyid}}", studyid) template = sub_template(template, "{{database}}", database) library = get_library(key='name') apps = "\n" # Add local experiments to library, first preference local_installs = 0 for experiment in experiments: if os.path.exists(experiment): bot.info('local experiment %s found, validating...' % experiment) # Is the experiment valid? cli = ExperimentValidator() valid = cli.validate(experiment, cleanup=False) if valid is True: local_installs += 1 config = load_experiment(experiment) exp_id = config['exp_id'] # If we aren't building in the experiment directory, we need to copy there output_dir = "%s/%s" % (os.path.abspath( os.path.dirname(args.output)), exp_id) experiment_dir = os.path.abspath(experiment) if output_dir != experiment_dir: copy_directory(experiment_dir, output_dir) config['local'] = os.path.abspath(experiment) library[exp_id] = config # Warn the user that local installs are not reproducible (from recipe) if local_installs > 0: bot.warning( "%s local installs detected: build is not reproducible without experiment folders" % local_installs) # Build Image with Experiments for experiment in experiments: exp_id = os.path.basename(experiment) if exp_id in library: config = library[exp_id] app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id) # Here add custom build routine, should be list of lines if "install" in config: commands = "\n".join( ["RUN %s " % s for x in config['install']]).strip('\n') app = "%s%s\n" % (app, commands) # The final installation step, either from Github (url) or local folder if "local" in config: app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id) app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % ( app, exp_id) else: app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % ( app, config['github']) apps = "%s%s\n" % (apps, app) else: bot.warning('%s not in library, check spelling & punctuation.' % exp_id) if apps == "\n": bot.error('No valid experiments found, cancelling recipe build.') sys.exit(1) template = sub_template(template, "{{experiments}}", apps) outfile = write_file(args.output, template) bot.log("Recipe written to %s" % outfile)
def main(): parser = get_parser() subparsers = get_subparsers(parser) try: args = parser.parse_args() except: sys.exit(0) # Does the use want to install? command = args.command # Options that shouldn't produce output if command in ['users']: os.environ['MESSAGELEVEL'] = "0" if args.database is not None: os.environ['EXPFACTORY_DATABASE'] = args.database from expfactory.logger import bot from expfactory.version import __version__ # The user may just want to parse the version if command == "version": print(__version__) sys.exit(0) bot.info("Expfactory Version: %s" % __version__) if command == "install": from .install import main elif command == "list": from .list import main elif command == "logs": from .logs import main elif command == "users": from .users import main elif command == "build": from .build import main # No argument supplied else: # A base exists for experiments base = os.environ.get('EXPFACTORY_BASE') if args.base is not None or base is not None: from .main import main command = "main" else: command = "list" from .list import main # Pass on to the correct parser if command is not None: # Main doesn't have a subparser subparser = None if command != "main": subparser = subparsers[command] main(args=args, parser=parser, subparser=subparser)
def start(port=5000, debug=False): bot.info("Nobody ever comes in... nobody ever comes out...") app.run(host="localhost", debug=debug, port=port)
def main(args,parser,subparser): template = 'build/docker/Dockerfile.template' # Full path to template is required if provided via input if args.input is not None: template = args.input template = get_template(template) # For now, only one database provided database = args.database studyid = args.studyid experiments = args.experiments branch = "-b %s" %os.environ.get('EXPFACTORY_BRANCH','master') headless = "false" if args.headless is True: headless = "true" template = sub_template(template,"{{studyid}}",studyid) template = sub_template(template,"{{database}}",database) template = sub_template(template,"{{headless}}",headless) template = sub_template(template,"{{branch}}",branch) if args.headless is True: bot.info("Headless build detected, you will need to generate tokens for application entry with expfactory users --new") library = get_library(key='name') apps = "\n" # Add local experiments to library, first preference local_installs = 0 for experiment in experiments: if os.path.exists(experiment): bot.info('local experiment %s found, validating...' %experiment) # Is the experiment valid? cli = ExperimentValidator() valid = cli.validate(experiment, cleanup=False) if valid is True: local_installs +=1 config = load_experiment(experiment) exp_id = config['exp_id'] # If we aren't building in the experiment directory, we need to copy there output_dir = "%s/%s" %(os.path.abspath(os.path.dirname(args.output)), exp_id) experiment_dir = os.path.abspath(experiment) if output_dir != experiment_dir: copy_directory(experiment_dir, output_dir) config['local'] = os.path.abspath(experiment) library[exp_id] = config # Warn the user that local installs are not reproducible (from recipe) if local_installs > 0: bot.warning("%s local installs detected: build is not reproducible without experiment folders" %local_installs) # Build Image with Experiments for experiment in experiments: exp_id = os.path.basename(experiment) if exp_id in library: config = library[exp_id] app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" %(exp_id, exp_id) # Here add custom build routine, should be list of lines if "install" in config: commands = "\n".join(["RUN %s "%s for x in config['install']]).strip('\n') app = "%s%s\n" %(app, commands) # The final installation step, either from Github (url) or local folder if "local" in config: app = "%sADD %s /scif/apps/%s\n" %(app, exp_id, exp_id) app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, exp_id) else: app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, config['github']) apps = "%s%s\n" %(apps,app) else: bot.warning('%s not in library, check spelling & punctuation.' %exp_id) if apps == "\n": bot.error('No valid experiments found, cancelling recipe build.') sys.exit(1) template = sub_template(template,"{{experiments}}",apps) outfile = write_file(args.output,template) bot.log("Recipe written to %s" %outfile)