def prepare_simulation(sim):
"""
Prepare a simulation given an incomplete row.
"""
spot = settings.SIMSPOT + sim.path
if os.path.isfile(spot):
return HttpResponse('control failure: path exists: %s' % spot)
conf = ortho.read_config()
ortho.sync(
modules={
spot:
conf.get(
'automacs', {
'address': 'https://github.com/biophyscode/automacs',
'branch': 'ortho'
})
})
bash('make', cwd=spot)
if False:
# copy a user-supplied gromacs_config.py if one is available
if settings.GROMACS_CONFIG:
# use the expected name for the local config
shutil.copyfile(
settings.GROMACS_CONFIG,
os.path.join(settings.SIMSPOT, sim.path, 'gromacs_config.py'))
elif not os.path.isfile(os.path.expanduser('~/.automacs.py')):
# if no global config, then we write one. this only happens once
# users who wish to customize the runs should edit ~/.automacs.py OR set an explicit gromacs_config
# key in the connect file which should point to a local copy of the .automacs.py
bash('make gromacs_config home',
cwd=settings.SIMSPOT + sim.path,
catch=True)
sim.save()
def connect(self,project_name,calc_spot,post_spot,settings_custom,
plot_spot,sims=None,calculations=None,public=False,development=True):
"""Basic connection from the factory to omnicalc."""
self.config = read_config()
# make directories minus calc spot which is cloned
dns = [post_spot,plot_spot]+([sims] if sims!=None else [])
for dn in dns:
if dn==None: continue
if not os.path.isdir(dn): mkdirs(dn)
calc_root = os.path.join('calc',project_name)
# clone omnicalc into the calculation folder
modules_this = {calc_root:
self.config.get('omnicalc',default_omnicalc_repo)}
try: modules.sync(modules=modules_this,current=True)
except Exception as e:
print('warning got error on sync: '+str(e))
print('warning failed to sync the repo: %s'%modules_this)
# clone and synchronize calculations reposotory
if calculations:
modules_calcs_this = {
os.path.join('calc',project_name,'calcs'):calculations}
try: modules.sync(modules=modules_calcs_this)
except: print('warning failed to sync the repo: %s'%
modules_calcs_this)
# update the postprocssing locations
ortho.bash('make set post_plot_spot %s'%plot_spot,cwd=calc_root)
ortho.bash('make set post_data_spot %s'%post_spot,cwd=calc_root)
# prepare the site, with some yaml keys passed through
#! could also pass through database, calc
site_setup(project_name,settings_custom=settings_custom,
public=public,development=development)
def connect(self,project_name,calc_spot,post_spot,settings_custom,
plot_spot,sims=None,calculations=None,public=False,development=True):
"""Basic connection from the factory to omnicalc."""
self.config = read_config()
# make directories minus calc spot which is cloned
dns = [post_spot,plot_spot]+([sims] if sims!=None else [])
for dn in dns:
if dn==None: continue
if not os.path.isdir(dn): mkdirs(dn)
calc_root = os.path.join('calc',project_name)
# clone omnicalc into the calculation folder
modules_this = {calc_root:
self.config.get('omnicalc',default_omnicalc_repo)}
try: modules.sync(modules=modules_this,current=True)
except Exception as e:
print('warning got error on sync: '+str(e))
print('warning failed to sync the repo: %s'%modules_this)
# clone and synchronize calculations reposotory
if calculations:
modules_calcs_this = {
os.path.join('calc',project_name,'calcs'):calculations}
try: modules.sync(modules=modules_calcs_this)
except: print('warning failed to sync the repo: %s'%
modules_calcs_this)
# update the postprocssing locations
ortho.bash('make set post_plot_spot %s'%plot_spot,cwd=calc_root)
ortho.bash('make set post_data_spot %s'%post_spot,cwd=calc_root)
# prepare the site, with some yaml keys passed through
#! could also pass through database, calc
site_setup(project_name,settings_custom=settings_custom,
public=public,development=development)
def test_make(self):
"""Make a blank config and ensure that it matches bootstrap and also test bootstrap."""
self.assertFalse(os.path.isfile(config_fn))
ortho.bash('make')
self.conf_start = ortho.read_config() #! ignore this
bootstrap = {}
exec(open('bootstrap.py', 'r').read(), bootstrap, bootstrap)
self.assertIn('default_configuration', bootstrap)
default_conf = bootstrap.get('default_configuration', {})
self.assertTrue(self.conf_start == default_conf)
self.assertIn('bootstrap_default', bootstrap)
def test_make(self):
"""Make a blank config and ensure that it matches bootstrap and also test bootstrap."""
self.assertFalse(os.path.isfile(config_fn))
ortho.bash('make')
self.conf_start = ortho.read_config() #! ignore this
bootstrap = {}
exec(open('bootstrap.py','r').read(),bootstrap,bootstrap)
self.assertIn('default_configuration',bootstrap)
default_conf = bootstrap.get('default_configuration',{})
self.assertTrue(self.conf_start==default_conf)
self.assertIn('bootstrap_default',bootstrap)
def _generate_programmatic_tests(self):
"""Special function noticed by suiteFactory to programatically add tests."""
# since we need to generate programmatic tests, we also run the typical setUp here as well
# ensure that we have a config.json
ortho.bash('make')
# make sure the config.json has the environments
self.prepare_environs()
# setUp is done here
# get the configuration and prepare unit tests for each environment
conf = ortho.read_config()
for env_name,detail in conf.get('envs',{}).items():
def this_test(self):
# note that you don't get variables from parent scope without refering to them (obv)
env_shortname = detail['name']
# rather than expecting a reqs.yaml, we use a single reqs_basic for all environment tests
#! should the requirements vary by environment in the test? real use-cases may differ
req_file = tempfile.NamedTemporaryFile(delete=False,suffix='json')
req_file.write(json.dumps(reqs_basic))
req_file.close()
# replace requirements. note that we check for installer and reqs above
conf['envs'][env_name]['sources']['reqs'] = req_file.name
custom_spot = tempfile.mkdtemp(prefix='env_%s_'%env_name,dir='.')
conf['envs'][env_name]['where'] = custom_spot
# use the update flag to install in a preexisting temporary directory
conf['envs'][env_name]['update'] = True
ortho.write_config(conf)
ortho.bash('make env %s'%env_name)
ortho.bash('make set activate_env="%s %s"'%(os.path.join(custom_spot,
'bin','activate'),env_shortname))
# conda is quirky. it does not appear in the path but runs after you source the env
# ... (possibly alias?) hence we have to use an absolute path to conda and also source
# ... the environment to check it we check on conda by using activate_env to ensure that
# ... this test is true-to-use
test_fn = 'test_commands.py'
#! this is conda specific. the whole test should be conda specific!
with open(test_fn,'w') as fp: fp.write(re.sub('WHERE_CONDA',
os.path.join(custom_spot,'bin','conda'),dummy_function_conda_list))
ortho.bash('make setlist commands %s'%test_fn)
# note ortho.bash fails to find conda in the conda_list dummy function
# note also that `result = ortho.bash('./env/bin/conda list')` does not source the
# ... right environment
result = ortho.bash('make conda_list',scroll=False)
# or `result = ortho.bash('source env/bin/activate py2 && ./env/bin/conda list',scroll=False)`
# check for some packages in reqs.yaml
reqs = reqs_basic['dependencies']
for key in reqs: self.assertIsNotNone(re.search(key,result.get('stdout')))
# clean up the test commands. note that config.json is replaced by the unittester
os.remove(test_fn)
return
# attach the tests to the class
func_name = str('test_environ_%s'%env_name)
this_test.__name__ = func_name
setattr(self,this_test.__name__,this_test)
def test_add_command(self):
"""Add a command."""
test_fn = 'test_commands.py'
self.assertFalse(os.path.isfile(test_fn))
with open(test_fn,'w') as fp: fp.write(dummy_function)
ortho.bash('make setlist commands %s'%test_fn)
conf = ortho.read_config() #! ignore this
self.assertTrue(test_fn in conf.get('commands',[]))
result = ortho.bash('make dummy',scroll=False)
# best to use bytes in patterns from now one
self.assertIsNotNone(re.search(b'special code',result.get('stdout')))
os.remove(test_fn)
def test_add_command(self):
"""Add a command."""
test_fn = 'test_commands.py'
self.assertFalse(os.path.isfile(test_fn))
with open(test_fn, 'w') as fp:
fp.write(dummy_function)
ortho.bash('make setlist commands %s' % test_fn)
conf = ortho.read_config() #! ignore this
self.assertTrue(test_fn in conf.get('commands', []))
result = ortho.bash('make dummy', scroll=False)
# best to use bytes in patterns from now one
self.assertIsNotNone(re.search(b'special code', result.get('stdout')))
os.remove(test_fn)
def prepare_environs(self):
"""Prepare the environment list."""
# control the available environments, otherwise set in environments.py
conf = ortho.read_config()
from ortho.environments import default_envs
conf['envs'] = default_envs
# make sure we have an installer and a requirements in each environment
for env_name,this in conf['envs'].items():
missing_keys = [k for k in ['installer','reqs']
if k not in this.get('sources',[])]
if any(missing_keys): raise Exception('environment %s is missing sources: %s'%(
env_name,missing_keys))
ortho.write_config(conf)
ortho.bash('make env list text=True')
def connect_run(project_name,settings_custom,post_spot,plot_spot,calculations,
public=False,development=False,sims=None,meta_filter=None,omnicalc=None):
"""
Instantiate a connection. Called by OmniFromFactory.
"""
config = read_config()
# make directories minus calc spot which is cloned
dns = [post_spot,plot_spot]+([sims] if sims!=None else [])
for dn in dns:
if dn==None: continue
if not os.path.isdir(dn): mkdirs(dn)
calc_root = os.path.join('calc',project_name)
# clone omnicalc into the calculation folder
modules_this = {calc_root:omnicalc if omnicalc else
config.get('omnicalc',default_omnicalc_repo)}
try: modules.sync(modules=modules_this,current=True)
except Exception as e:
print('warning got error on sync: '+str(e))
print('warning failed to sync the repo: %s'%modules_this)
# clone and synchronize calculations reposotory
if calculations:
modules_calcs_this = {
os.path.join('calc',project_name,'calcs'):calculations}
try: modules.sync(modules=modules_calcs_this,current=True)
except: print('warning failed to sync the repo: %s'%
modules_calcs_this)
# absolute paths
plot_spot = os.path.realpath(plot_spot)
post_spot = os.path.realpath(post_spot)
# update the postprocssing locations
ortho.bash('make set post_plot_spot %s'%plot_spot,cwd=calc_root)
ortho.bash('make set post_data_spot %s'%post_spot,cwd=calc_root)
if meta_filter:
ortho.bash('make unset meta_filter',cwd=calc_root)
meta_filter = ortho.listify(meta_filter)
ortho.bash('make setlist meta_filter %s'%
' '.join(meta_filter),cwd=calc_root)
# prepare the site, with some yaml keys passed through
#! could also pass through database, calc
site_setup(project_name,settings_custom=settings_custom,
public=public,development=development)
# propagate the flag to activate the environment
spec_py3 = config.get('installed',{}).get('conda_py3',{})
if spec_py3:
activate_path = os.path.realpath(
os.path.join(spec_py3['where'],'bin','activate'))
ortho.bash('make set activate_env="%s py3"'%activate_path,cwd=calc_root)
#! currently we only pass the conda_py3 environment
else: pass
def connection_template(kind,name):
"""
Make a template and write the file.
:kind: a style (basic)
:name: the name of the project
"""
config = read_config()
connection_templates = {
'basic':{
'calc_spot':'calc/PROJECT_NAME',
'post_spot':'data/PROJECT_NAME/post',
'plot_spot':'plot/PROJECT_NAME/plot',},}
template_handler = config.get(
'connection_to_template',connection_to_template)
hook = Hook(**template_handler)
hook.function(name=name,specs=connection_templates[kind])
def connection_template(kind,name):
"""
Make a template and write the file.
:kind: a style (basic)
:name: the name of the project
"""
config = read_config()
connection_templates = {
'basic':{
'calc_spot':'calc/PROJECT_NAME',
'post_spot':'data/PROJECT_NAME/post',
'plot_spot':'plot/PROJECT_NAME/plot',},}
template_handler = config.get(
'connection_to_template',connection_to_template)
hook = Hook(**template_handler)
hook.function(name=name,specs=connection_templates[kind])
def prepare_environs(self):
"""Prepare the environment list."""
# control the available environments, otherwise set in environments.py
conf = ortho.read_config()
from ortho.environments import default_envs
conf['envs'] = default_envs
# make sure we have an installer and a requirements in each environment
for env_name, this in conf['envs'].items():
missing_keys = [
k for k in ['installer', 'reqs']
if k not in this.get('sources', [])
]
if any(missing_keys):
raise Exception('environment %s is missing sources: %s' %
(env_name, missing_keys))
ortho.write_config(conf)
ortho.bash('make env list text=True')
def bootstrap(post=True, refresh=False):
"""
This function runs setup commands from `bootstrap.py`.
It is automatically interpreted from read_config if config.json is absent. If `config.json` is missing
and we find bootstrap.py and it supplies a function called `bootstrap_default()` then that function's
return value serves as the default configuration.
After writing the default configuration, we run `bootstrap_post()` if it is available. This function also
runs on any repeated execution of `make bootstrap` from the command line.
"""
import importlib, os, sys
if not os.path.isfile('bootstrap.py'): return None
# import to run the script
if (sys.version_info < (3, 0)):
mod = importlib.import_module('bootstrap', package='.')
else:
mod = importlib.import_module('bootstrap', package=None)
has_bootstrap_default = hasattr(mod, 'bootstrap_default')
has_bootstrap_post = hasattr(mod, 'bootstrap_post')
if not has_bootstrap_post and not has_bootstrap_default:
# this warning is invisible when ortho runs for the first time to make config.json but
# ... read_config will warn the user if these functions are both
print(
'warning',
'the local bootstrap.py lacks functions bootstrap_default or boostrap_post'
)
outgoing = {}
if has_bootstrap_default: outgoing['default'] = mod.bootstrap_default()
if has_bootstrap_post: outgoing['post'] = mod.bootstrap_post
# when not initializing we skip the default configuration and just run the post
#! this is somewhat arbitrary. we may wish to reconfigure on make bootstrap using the default
if post and has_bootstrap_post: mod.bootstrap_post()
# read_config only collects post and runs it after setting up default configuration
# note that this function returns to read_config for making the config if one is absent
# however you may also use it to push changes from bootstrap.py to an existing config.json
# by running `make bootstrap refresh` after updating bootstrap.py
if refresh:
if not os.path.isfile(config_fn):
raise Exception(
'cannot refresh from bootstrap because %s is missing' %
config_fn)
from ortho import read_config, write_config
conf = read_config()
conf.update(**outgoing.get('default', {}))
write_config(conf)
return
return outgoing
def get_connections(name):
"""
Get connections via hook.
"""
config = read_config()
connection_handler = config.get(
'connection_handler',default_connection_handler)
hook = Hook(**connection_handler)
toc = hook.function()
if name not in toc:
treeview(dict(connections=toc.keys()))
raise Exception(
'cannot find connection in the list of connections above: %s'%name)
specs = toc[name]
# multiple connections can map to the same project by setting the name here
name = specs.pop('name',name)
specs['project_name'] = name
return specs
def get_connections(name):
"""
Get connections via hook.
"""
config = read_config()
connection_handler = config.get(
'connection_handler',default_connection_handler)
hook = Hook(**connection_handler)
toc = hook.function()
if name not in toc:
treeview(dict(connections=toc.keys()))
raise Exception(
'cannot find connection in the list of connections above: %s'%name)
specs = toc[name]
# multiple connections can map to the same project by setting the name here
name = specs.pop('name',name)
specs['project_name'] = name
return specs
def connect_run(project_name,settings_custom,post_spot,plot_spot,calculations,
public=False,development=False,sims=None,meta_filter=None):
"""
Instantiate a connection. Called by OmniFromFactory.
"""
config = read_config()
# make directories minus calc spot which is cloned
dns = [post_spot,plot_spot]+([sims] if sims!=None else [])
for dn in dns:
if dn==None: continue
if not os.path.isdir(dn): mkdirs(dn)
calc_root = os.path.join('calc',project_name)
# clone omnicalc into the calculation folder
modules_this = {calc_root:
config.get('omnicalc',default_omnicalc_repo)}
try: modules.sync(modules=modules_this,current=True)
except Exception as e:
print('warning got error on sync: '+str(e))
print('warning failed to sync the repo: %s'%modules_this)
# clone and synchronize calculations reposotory
if calculations:
modules_calcs_this = {
os.path.join('calc',project_name,'calcs'):calculations}
try: modules.sync(modules=modules_calcs_this,current=True)
except: print('warning failed to sync the repo: %s'%
modules_calcs_this)
# absolute paths
plot_spot = os.path.realpath(plot_spot)
post_spot = os.path.realpath(post_spot)
# update the postprocssing locations
ortho.bash('make set post_plot_spot %s'%plot_spot,cwd=calc_root)
ortho.bash('make set post_data_spot %s'%post_spot,cwd=calc_root)
if meta_filter:
ortho.bash('make unset meta_filter',cwd=calc_root)
meta_filter = ortho.listify(meta_filter)
ortho.bash('make setlist meta_filter %s'%
' '.join(meta_filter),cwd=calc_root)
# prepare the site, with some yaml keys passed through
#! could also pass through database, calc
site_setup(project_name,settings_custom=settings_custom,
public=public,development=development)
def bootstrap(post=True,refresh=False):
"""
This function runs setup commands from `bootstrap.py`.
It is automatically interpreted from read_config if config.json is absent. If `config.json` is missing
and we find bootstrap.py and it supplies a function called `bootstrap_default()` then that function's
return value serves as the default configuration.
After writing the default configuration, we run `bootstrap_post()` if it is available. This function also
runs on any repeated execution of `make bootstrap` from the command line.
"""
import importlib,os,sys
if not os.path.isfile('bootstrap.py'): return None
# import to run the script
if (sys.version_info < (3, 0)): mod = importlib.import_module('bootstrap',package='.')
else: mod = importlib.import_module('bootstrap',package=None)
has_bootstrap_default = hasattr(mod,'bootstrap_default')
has_bootstrap_post = hasattr(mod,'bootstrap_post')
if not has_bootstrap_post and not has_bootstrap_default:
# this warning is invisible when ortho runs for the first time to make config.json but
# ... read_config will warn the user if these functions are both
print('warning','the local bootstrap.py lacks functions bootstrap_default or boostrap_post')
outgoing = {}
if has_bootstrap_default: outgoing['default'] = mod.bootstrap_default()
if has_bootstrap_post: outgoing['post'] = mod.bootstrap_post
# when not initializing we skip the default configuration and just run the post
#! this is somewhat arbitrary. we may wish to reconfigure on make bootstrap using the default
if post and has_bootstrap_post: mod.bootstrap_post()
# read_config only collects post and runs it after setting up default configuration
# note that this function returns to read_config for making the config if one is absent
# however you may also use it to push changes from bootstrap.py to an existing config.json
# by running `make bootstrap refresh` after updating bootstrap.py
if refresh:
if not os.path.isfile(config_fn):
raise Exception('cannot refresh from bootstrap because %s is missing'%config_fn)
from ortho import read_config,write_config
conf = read_config()
conf.update(**outgoing.get('default',{}))
write_config(conf)
return
return outgoing
def _generate_programmatic_tests(self):
"""Special function noticed by suiteFactory to programatically add tests."""
# since we need to generate programmatic tests, we also run the typical setUp here as well
# ensure that we have a config.json
ortho.bash('make')
# make sure the config.json has the environments
self.prepare_environs()
# setUp is done here
# get the configuration and prepare unit tests for each environment
conf = ortho.read_config()
for env_name, detail in conf.get('envs', {}).items():
def this_test(self):
# note that you don't get variables from parent scope without refering to them (obv)
env_shortname = detail['name']
# rather than expecting a reqs.yaml, we use a single reqs_basic for all environment tests
#! should the requirements vary by environment in the test? real use-cases may differ
req_file = tempfile.NamedTemporaryFile(delete=False,
suffix='json')
req_file.write(json.dumps(reqs_basic))
req_file.close()
# replace requirements. note that we check for installer and reqs above
conf['envs'][env_name]['sources']['reqs'] = req_file.name
custom_spot = tempfile.mkdtemp(prefix='env_%s_' % env_name,
dir='.')
conf['envs'][env_name]['where'] = custom_spot
# use the update flag to install in a preexisting temporary directory
conf['envs'][env_name]['update'] = True
ortho.write_config(conf)
ortho.bash('make env %s' % env_name)
ortho.bash('make set activate_env="%s %s"' % (os.path.join(
custom_spot, 'bin', 'activate'), env_shortname))
# conda is quirky. it does not appear in the path but runs after you source the env
# ... (possibly alias?) hence we have to use an absolute path to conda and also source
# ... the environment to check it we check on conda by using activate_env to ensure that
# ... this test is true-to-use
test_fn = 'test_commands.py'
#! this is conda specific. the whole test should be conda specific!
with open(test_fn, 'w') as fp:
fp.write(
re.sub('WHERE_CONDA',
os.path.join(custom_spot, 'bin', 'conda'),
dummy_function_conda_list))
ortho.bash('make setlist commands %s' % test_fn)
# note ortho.bash fails to find conda in the conda_list dummy function
# note also that `result = ortho.bash('./env/bin/conda list')` does not source the
# ... right environment
result = ortho.bash('make conda_list', scroll=False)
# or `result = ortho.bash('source env/bin/activate py2 && ./env/bin/conda list',scroll=False)`
# check for some packages in reqs.yaml
reqs = reqs_basic['dependencies']
for key in reqs:
self.assertIsNotNone(re.search(key, result.get('stdout')))
# clean up the test commands. note that config.json is replaced by the unittester
os.remove(test_fn)
return
# attach the tests to the class
func_name = str('test_environ_%s' % env_name)
this_test.__name__ = func_name
setattr(self, this_test.__name__, this_test)