def init_db(self):
'''initialize the database, with the default database path or custom of
the format sqlite:////scif/data/expfactory.db
'''
# Database Setup, use default if uri not provided
if self.database == 'sqlite':
db_path = os.path.join(EXPFACTORY_DATA, '%s.db' % EXPFACTORY_SUBID)
self.database = 'sqlite:///%s' % db_path
bot.info("Database located at %s" % self.database)
self.engine = create_engine(self.database, convert_unicode=True)
self.session = scoped_session(sessionmaker(autocommit=False,
autoflush=False,
bind=self.engine))
Base.query = self.session.query_property()
# import all modules here that might define models so that
# they will be registered properly on the metadata. Otherwise
# you will have to import them first before calling init_db()
import expfactory.database.models
Base.metadata.create_all(bind=self.engine)
self.Base = Base
def main():
parser = get_parser()
subparsers = get_subparsers(parser)
try:
args = parser.parse_args()
except:
sys.exit(0)
# Does the use want to install?
command = args.command
# Options that shouldn't produce output
if command in ['users']:
os.environ['MESSAGELEVEL'] = "0"
if args.database is not None:
os.environ['EXPFACTORY_DATABASE'] = args.database
from expfactory.logger import bot
from expfactory.version import __version__
bot.info("Expfactory Version: %s" % __version__)
if command == "install":
from .install import main
elif command == "list":
from .list import main
elif command == "logs":
from .logs import main
elif command == "users":
from .users import main
elif command == "build":
from .build import main
# No argument supplied
else:
# A base exists for experiments
base = os.environ.get('EXPFACTORY_BASE')
if args.base is not None or base is not None:
from .main import main
command = "main"
else:
command = "list"
from .list import main
# Pass on to the correct parser
if command is not None:
# Main doesn't have a subparser
subparser = None
if command != "main":
subparser = subparsers[command]
main(args=args, parser=parser, subparser=subparser)
def init_db(self):
'''initialize the database, with the default database path or custom of
the format sqlite:////scif/data/expfactory.db
'''
# Database Setup, use default if uri not provided
if self.database == 'sqlite':
db_path = os.path.join(EXPFACTORY_DATA, '%s.db' % EXPFACTORY_SUBID)
self.database = 'sqlite:///%s' % db_path
bot.info("Database located at %s" % self.database)
self.engine = create_engine(self.database, convert_unicode=True)
self.session = scoped_session(sessionmaker(autocommit=False,
autoflush=False,
bind=self.engine))
Base.query = self.session.query_property()
# import all modules here that might define models so that
# they will be registered properly on the metadata. Otherwise
# you will have to import them first before calling init_db()
import expfactory.database.models
Base.metadata.create_all(bind=self.engine)
self.Base = Base
def main(args, parser, subparser):
from expfactory.server import app
header = 'DATABASE\tTOKEN'
# The user wants to list active subjects
if args.list is True:
users = app.list_users() # returns id\ttoken
sys.exit(0)
# The user wants to add new subjects
number = args.new
if number is not None:
print(header)
for i in range(number):
user = app.generate_user()
app.print_user(user)
sys.exit(0)
# The user wants to manage user token
action = None
if args.revoke is not None:
subid = clean(args.revoke)
func = app.revoke_token
action = "Revoking"
elif args.refresh is not None:
subid = clean(args.refresh)
func = app.refresh_token
action = "Refreshing"
elif args.restart is not None:
subid = clean(args.restart)
func = app.restart_user
action = "Restarting"
elif args.finish is not None:
subid = clean(args.finish)
action = "Finishing"
func = app.finish_user
# Perform the action
if action is not None:
bot.info('%s %s' %(action, subid))
result = func(subid=subid)
if result is not None:
print("[%s] %s --> %s" %(action.lower(),
subid,
result))
else:
print("[%s] not successful. See logs for details." %(action.lower()))
print("Commands may only possible for [active] status.")
sys.exit(0)
print('See expfactory users --help for usage')
def main(args, parser, subparser):
response = requests.get(EXPFACTORY_LIBRARY)
if response.status_code == 200:
library = response.json()
bot.info("Experiments")
rows = []
for experiment in library:
rows.append([experiment["name"], experiment["github"]])
bot.table(rows)
def main(args,parser,subparser):
response = requests.get(EXPFACTORY_LIBRARY)
if response.status_code == 200:
library = response.json()
bot.info("Experiments")
rows = []
for experiment in library:
rows.append([ experiment['name'],
experiment['github'] ])
bot.table(rows)
def get_experiments(base, load=False):
""" get_experiments will return loaded json for all valid experiments from an experiment folder
:param base: full path to the base folder with experiments inside
:param load: if True, returns a list of loaded config.json objects. If False (default) returns the paths to the experiments
"""
experiments = find_directories(base)
valid_experiments = [e for e in experiments if validate(e, cleanup=False)]
bot.info("Found %s valid experiments" % (len(valid_experiments)))
if load is True:
valid_experiments = load_experiments(valid_experiments)
# TODO at some point in this workflow we would want to grab instructions from help
# and variables from labels, environment, etc.
return valid_experiments
def get_experiments(base, load=False):
''' get_experiments will return loaded json for all valid experiments from an experiment folder
:param base: full path to the base folder with experiments inside
:param load: if True, returns a list of loaded config.json objects. If False (default) returns the paths to the experiments
'''
experiments = find_directories(base)
valid_experiments = [e for e in experiments if validate(e,cleanup=False)]
bot.info("Found %s valid experiments" %(len(valid_experiments)))
if load is True:
valid_experiments = load_experiments(valid_experiments)
#TODO at some point in this workflow we would want to grab instructions from help
# and variables from labels, environment, etc.
return valid_experiments
def main():
parser = get_parser()
try:
args = parser.parse_args()
except:
sys.exit(0)
folder = args.folder
if folder is None:
folder = os.getcwd()
folder = os.path.abspath(folder)
survey = "%s/survey.tsv" % folder
if not os.path.exists(survey):
bot.error("Cannot find %s, required to generate survey." % survey)
sys.exit(1)
config = load_experiment(folder)
html, validation = generate_survey(config=config,
survey_file=survey,
form_action=args.action)
output = args.output
if output is None:
output = folder
output_index = "%s/index.html" % folder
if os.path.exists(output_index) and args.force is False:
bot.error("%s already exists, use --force to overwrite." %
output_index)
sys.exit(1)
bot.info("Writing output files to %s" % output_index)
template = get_template('survey/index.html')
template = sub_template(template, "{{html}}", html)
template = sub_template(template, "{{exp_id}}", config['exp_id'])
template = sub_template(template, "{{validation}}", validation)
write_file(output_index, template)
def initdb(self):
'''initdb will check for writability of the data folder, meaning
that it is bound to the local machine. If the folder isn't bound,
expfactory runs in demo mode (not saving data)
'''
self.database = EXPFACTORY_DATABASE
bot.info("DATABASE: %s" %self.database)
# Supported database options
valid = ('sqlite', 'postgres', 'mysql', 'filesystem')
if not self.database.startswith(valid):
bot.warning('%s is not yet a supported type, saving to filesystem.' % self.database)
self.database = 'filesystem'
# Add functions specific to database type
self.init_db() # uses url in self.database
bot.log("Data base: %s" % self.database)
def initdb(self):
'''initdb will check for writability of the data folder, meaning
that it is bound to the local machine. If the folder isn't bound,
expfactory runs in demo mode (not saving data)
'''
self.database = EXPFACTORY_DATABASE
bot.info("DATABASE: %s" %self.database)
# Supported database options
valid = ('sqlite', 'postgres', 'mysql', 'filesystem')
if not self.database.startswith(valid):
bot.warning('%s is not yet a supported type, saving to filesystem.' % self.database)
self.database = 'filesystem'
# Add functions specific to database type
self.init_db() # uses url in self.database
bot.log("Data base: %s" % self.database)
def save_data(self, session, exp_id, content):
'''save data will obtain the current subid from the session, and save it
depending on the database type.'''
from expfactory.database.models import (Participant, Result)
subid = session.get('subid')
bot.info('Saving data for subid %s' % subid)
# We only attempt save if there is a subject id, set at start
if subid is not None:
p = Participant.query.filter(
Participant.id == subid).first() # better query here
# Preference is to save data under 'data', otherwise do all of it
if "data" in content:
content = content['data']
if isinstance(content, dict):
content = json.dumps(content)
result = Result(data=content, exp_id=exp_id,
participant_id=p.id) # check if changes from str/int
self.session.add(result)
p.results.append(result)
self.session.commit()
bot.info("Participant: %s" % p)
bot.info("Result: %s" % result)
def save_data(self,session, exp_id, content):
'''save data will obtain the current subid from the session, and save it
depending on the database type.'''
from expfactory.database.models import (
Participant,
Result
)
subid = session.get('subid')
bot.info('Saving data for subid %s' % subid)
# We only attempt save if there is a subject id, set at start
if subid is not None:
p = Participant.query.filter(Participant.id == subid).first() # better query here
# Preference is to save data under 'data', otherwise do all of it
if "data" in content:
content = content['data']
if isinstance(content,dict):
content = json.dumps(content)
result = Result(data=content,
exp_id=exp_id,
participant_id=p.id) # check if changes from str/int
self.session.add(result)
p.results.append(result)
self.session.commit()
bot.info("Participant: %s" %p)
bot.info("Result: %s" %result)
def main(args, parser, subparser):
template = "build/docker/Dockerfile.template"
# Full path to template is required if provided via input
if args.input is not None:
template = args.input
template = get_template(template)
# For now, only one database provided
database = args.database
studyid = args.studyid
experiments = args.experiments
branch = "-b %s" % os.environ.get("EXPFACTORY_BRANCH", "master")
headless = "false"
if args.headless is True:
headless = "true"
template = sub_template(template, "{{studyid}}", studyid)
template = sub_template(template, "{{database}}", database)
template = sub_template(template, "{{headless}}", headless)
template = sub_template(template, "{{branch}}", branch)
if args.headless is True:
bot.info(
"Headless build detected, you will need to generate tokens for application entry with expfactory users --new"
)
library = get_library(key="name")
apps = "\n"
# Add local experiments to library, first preference
local_installs = 0
for experiment in experiments:
if os.path.exists(experiment):
bot.info("local experiment %s found, validating..." % experiment)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(experiment, cleanup=False)
if valid is True:
local_installs += 1
config = load_experiment(experiment)
exp_id = config["exp_id"]
# If we aren't building in the experiment directory, we need to copy there
output_dir = "%s/%s" % (
os.path.abspath(os.path.dirname(args.output)),
exp_id,
)
experiment_dir = os.path.abspath(experiment)
if output_dir != experiment_dir:
copy_directory(experiment_dir, output_dir)
config["local"] = os.path.abspath(experiment)
library[exp_id] = config
# Warn the user that local installs are not reproducible (from recipe)
if local_installs > 0:
bot.warning(
"%s local installs detected: build is not reproducible without experiment folders"
% local_installs
)
# Build Image with Experiments
for experiment in experiments:
exp_id = os.path.basename(experiment)
if exp_id in library:
config = library[exp_id]
app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id)
# Here add custom build routine, should be list of lines
if "install" in config:
commands = "\n".join(["RUN %s " % s for x in config["install"]]).strip(
"\n"
)
app = "%s%s\n" % (app, commands)
# The final installation step, either from Github (url) or local folder
if "local" in config:
app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id)
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app,
exp_id,
)
else:
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app,
config["github"],
)
apps = "%s%s\n" % (apps, app)
else:
bot.warning("%s not in library, check spelling & punctuation." % exp_id)
if apps == "\n":
bot.error("No valid experiments found, cancelling recipe build.")
sys.exit(1)
template = sub_template(template, "{{experiments}}", apps)
outfile = write_file(args.output, template)
bot.log("Recipe written to %s" % outfile)
def main(args, parser, subparser):
# List of experiments is required
template = get_template('build/docker/Dockerfile.template')
# For now, only one database provided
database = args.database
studyid = args.studyid
experiments = args.experiments
template = sub_template(template, "{{studyid}}", studyid)
template = sub_template(template, "{{database}}", database)
library = get_library(key='name')
apps = "\n"
# Add local experiments to library, first preference
local_installs = 0
for experiment in experiments:
if os.path.exists(experiment):
bot.info('local experiment %s found, validating...' % experiment)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(experiment, cleanup=False)
if valid is True:
local_installs += 1
config = load_experiment(experiment)
exp_id = config['exp_id']
# If we aren't building in the experiment directory, we need to copy there
output_dir = "%s/%s" % (os.path.abspath(
os.path.dirname(args.output)), exp_id)
experiment_dir = os.path.abspath(experiment)
if output_dir != experiment_dir:
copy_directory(experiment_dir, output_dir)
config['local'] = os.path.abspath(experiment)
library[exp_id] = config
# Warn the user that local installs are not reproducible (from recipe)
if local_installs > 0:
bot.warning(
"%s local installs detected: build is not reproducible without experiment folders"
% local_installs)
# Build Image with Experiments
for experiment in experiments:
exp_id = os.path.basename(experiment)
if exp_id in library:
config = library[exp_id]
app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" % (exp_id, exp_id)
# Here add custom build routine, should be list of lines
if "install" in config:
commands = "\n".join(
["RUN %s " % s for x in config['install']]).strip('\n')
app = "%s%s\n" % (app, commands)
# The final installation step, either from Github (url) or local folder
if "local" in config:
app = "%sADD %s /scif/apps/%s\n" % (app, exp_id, exp_id)
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app, exp_id)
else:
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" % (
app, config['github'])
apps = "%s%s\n" % (apps, app)
else:
bot.warning('%s not in library, check spelling & punctuation.' %
exp_id)
if apps == "\n":
bot.error('No valid experiments found, cancelling recipe build.')
sys.exit(1)
template = sub_template(template, "{{experiments}}", apps)
outfile = write_file(args.output, template)
bot.log("Recipe written to %s" % outfile)
def main():
parser = get_parser()
subparsers = get_subparsers(parser)
try:
args = parser.parse_args()
except:
sys.exit(0)
# Does the use want to install?
command = args.command
# Options that shouldn't produce output
if command in ['users']:
os.environ['MESSAGELEVEL'] = "0"
if args.database is not None:
os.environ['EXPFACTORY_DATABASE'] = args.database
from expfactory.logger import bot
from expfactory.version import __version__
# The user may just want to parse the version
if command == "version":
print(__version__)
sys.exit(0)
bot.info("Expfactory Version: %s" % __version__)
if command == "install":
from .install import main
elif command == "list":
from .list import main
elif command == "logs":
from .logs import main
elif command == "users":
from .users import main
elif command == "build":
from .build import main
# No argument supplied
else:
# A base exists for experiments
base = os.environ.get('EXPFACTORY_BASE')
if args.base is not None or base is not None:
from .main import main
command = "main"
else:
command = "list"
from .list import main
# Pass on to the correct parser
if command is not None:
# Main doesn't have a subparser
subparser = None
if command != "main":
subparser = subparsers[command]
main(args=args,
parser=parser,
subparser=subparser)
def start(port=5000, debug=False):
bot.info("Nobody ever comes in... nobody ever comes out...")
app.run(host="localhost", debug=debug, port=port)
def main(args,parser,subparser):
template = 'build/docker/Dockerfile.template'
# Full path to template is required if provided via input
if args.input is not None:
template = args.input
template = get_template(template)
# For now, only one database provided
database = args.database
studyid = args.studyid
experiments = args.experiments
branch = "-b %s" %os.environ.get('EXPFACTORY_BRANCH','master')
headless = "false"
if args.headless is True:
headless = "true"
template = sub_template(template,"{{studyid}}",studyid)
template = sub_template(template,"{{database}}",database)
template = sub_template(template,"{{headless}}",headless)
template = sub_template(template,"{{branch}}",branch)
if args.headless is True:
bot.info("Headless build detected, you will need to generate tokens for application entry with expfactory users --new")
library = get_library(key='name')
apps = "\n"
# Add local experiments to library, first preference
local_installs = 0
for experiment in experiments:
if os.path.exists(experiment):
bot.info('local experiment %s found, validating...' %experiment)
# Is the experiment valid?
cli = ExperimentValidator()
valid = cli.validate(experiment, cleanup=False)
if valid is True:
local_installs +=1
config = load_experiment(experiment)
exp_id = config['exp_id']
# If we aren't building in the experiment directory, we need to copy there
output_dir = "%s/%s" %(os.path.abspath(os.path.dirname(args.output)), exp_id)
experiment_dir = os.path.abspath(experiment)
if output_dir != experiment_dir:
copy_directory(experiment_dir, output_dir)
config['local'] = os.path.abspath(experiment)
library[exp_id] = config
# Warn the user that local installs are not reproducible (from recipe)
if local_installs > 0:
bot.warning("%s local installs detected: build is not reproducible without experiment folders" %local_installs)
# Build Image with Experiments
for experiment in experiments:
exp_id = os.path.basename(experiment)
if exp_id in library:
config = library[exp_id]
app = "LABEL EXPERIMENT_%s /scif/apps/%s\n" %(exp_id, exp_id)
# Here add custom build routine, should be list of lines
if "install" in config:
commands = "\n".join(["RUN %s "%s for x in config['install']]).strip('\n')
app = "%s%s\n" %(app, commands)
# The final installation step, either from Github (url) or local folder
if "local" in config:
app = "%sADD %s /scif/apps/%s\n" %(app, exp_id, exp_id)
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, exp_id)
else:
app = "%sWORKDIR /scif/apps\nRUN expfactory install %s\n" %(app, config['github'])
apps = "%s%s\n" %(apps,app)
else:
bot.warning('%s not in library, check spelling & punctuation.' %exp_id)
if apps == "\n":
bot.error('No valid experiments found, cancelling recipe build.')
sys.exit(1)
template = sub_template(template,"{{experiments}}",apps)
outfile = write_file(args.output,template)
bot.log("Recipe written to %s" %outfile)
def start(port=5000, debug=False):
bot.info("Nobody ever comes in... nobody ever comes out...")
app.run(host="localhost", debug=debug, port=port)
