def __init__(self,fn,previous=False,autoreload=False):
"""
Note that we initialize paths.yaml and superficial data first, then load from the workspace or
parse the dataset if the workspace was not saved.
"""
self.filename = path_expand(fn)
self.paths = unpacker(conf_paths)
self.machine = unpacker(conf_gromacs,'machine_configuration')[machine_name]
del sys.modules['yaml']
self.nprocs = self.machine['nprocs'] if 'nprocs' in self.machine else 1
#---write timeseries to the post data directory when making slices
self.write_timeseries_to_disk = self.paths.get('timekeeper',False)
self.postdir = os.path.join(path_expand(self.paths['post_data_spot']),'')
self.plotdir = os.path.join(path_expand(self.paths['post_plot_spot']),'')
#---generic variables associated with the workspace
self.vars = {}
self.meta = {}
self.calc = {}
#---automatically add preexisting files to the workspace if they are found
self.autoreload = autoreload
#---for each "spot" in the yaml file, we construct a template for the data therein
#---the table of contents ("toc") holds one parsing for every part regex in every spot
self.spots,self.toc = {},collections.OrderedDict()
for name,details in self.paths['spots'].items():
rootdir = os.path.join(details['route_to_data'],details['spot_directory'])
if not os.path.isdir(rootdir):
raise Exception('\n[ERROR] cannot find root directory %s'%rootdir)
for part_name,part_regex in details['regexes']['part'].items():
spot = (name,part_name)
self.toc[spot] = {}
self.spots[spot] = {
'rootdir':os.path.join(rootdir,''),
'top':details['regexes']['top'],
'step':details['regexes']['step'],
'part':part_regex,
'namer':eval(details['namer']),
'namer_text':details['namer'],
}
self.spots[spot]['divy_keys'] = self.divy_keys(spot)
#---we always require an xtc entry in the parts list
if 'xtc' not in zip(*self.spots.keys())[1]:
raise Exception('\n[ERROR] you must have "xtc" in the parts list')
#---set a cursor which specifies the active spot which should always be the first in the yaml
self.cursor = self.spots.keys()[0]
#---the self.c variable holds the top spot name but not the part name
self.c = self.cursor[0]
#---! default to XTC
self.trajectory_format = 'xtc'
self.merge_method = self.paths.get('merge_method','careful')
#---open self if the filename exists
#---note that we save parser results but not details from paths.yaml in case these change
if os.path.isfile(self.filename):
self.load(previous=previous)
self.verify()
#---otherwise make a new workspace
else: self.bootstrap()
def plot(plotname=None,nox=False,workspace=None,specfile=None,plotlog=False,**kwargs):
"""
Run a plotting routine.
"""
from copy import deepcopy
if plotname == None:
from base.workspace import Workspace
if workspace == None: workspace = unpacker(conf_paths)['workspace_spot']
work = Workspace(workspace,previous=False)
specs = work.load_specs()
plotnames = specs['plots'].keys()
else: plotnames = [plotname]
#---for each desired plot type
for pname in plotnames:
fns = []
for (dirpath, dirnames, filenames) in os.walk('./'):
fns.extend([dirpath+'/'+fn for fn in filenames])
search = filter(lambda x:re.match('^\.\/[^omni].+\/plot-%s\.py$'%pname,x),fns)
if len(search)!=1: status('unclear search for %s: %s'%(pname,str(search)))
else:
if plotname==None:
cmd = 'python '+search[0]+' nox quit=True '+' "%s"'%str(kwargs)+\
(' &> %s'%plotlog if plotlog else '')
else:
status('rerun the plot with:\n\nexecfile(\''+search[0]+'\')\n',tag='note')
cmd = "python -i "+search[0]+(' nox' if nox else '')+' "%s"'%str(kwargs)
#---! add log here? is the user or factory ever going to use this?
status('calling: "%s"'%cmd,tag='status')
os.system(cmd)
def export_to_factory(project_name,project_location,workspace=None):
"""
Export the simulation data from the toc to the factory database.
Users should not run this.
"""
sys.path.insert(0,project_location)
os.environ.setdefault("DJANGO_SETTINGS_MODULE",project_name+".settings")
import django
#---! a fairly extreme hack. this function needs a lot of work. the paths are almost certainly fubar
sys.path.insert(0,os.path.join(os.path.abspath(os.path.join(project_location,'../../')),'dev'))
django.setup()
from simulator import models
from base.workspace import Workspace
workspace = unpacker(conf_paths)['workspace_spot']
work = Workspace(workspace)
sns={}
for key in work.toc.keys():
for sn in work.toc[key]:
sns[sn]=key[0]
for sn in sns.keys():
if any([sn in work.toc[i] for i in work.toc.keys() if i[1]=='edr']) and (
any([sn in work.toc[i] for i in work.toc.keys() if i[1]=='trr']) or
any([sn in work.toc[i] for i in work.toc.keys() if i[1]=='xtc'])):
spot=(sns[sn],'edr')
name=work.prefixer(sn)
try: models.Simulation(name=name,program="protein",code=sn).save()
except: print '[NOTE] simulation "%s" already exists in the database'%name
if not sns: print "[STATUS] nothing to export"
def refresh(autoreload=True): """ If you have new data or more data (i.e. more XTC files or longer trajectories) you must run refresh in order to add those files to the tables of contents. """ from base.workspace import Workspace workspace = unpacker(conf_paths)['workspace_spot'] work = Workspace(workspace,autoreload=autoreload) work.bootstrap() work.save()
def compute(calculation_name=None,autoreload=True): """ Open the workspace, parse a YAML script with instructions, save, and exit. Note that we may specify a particular calculation if there are many pending calculations. """ from base.workspace import Workspace workspace = unpacker(conf_paths)['workspace_spot'] work = Workspace(workspace,autoreload=autoreload) work.action(calculation_name=calculation_name) work.save()
def action(self,calculation_name=None):
"""
Parse a specifications file to make changes to a workspace.
This function interprets the specifications and acts on it.
It manages the irreducible units of an omnicalc operation and ensures
that the correct data are sent to analysis functions in the right order.
"""
status('parsing specs file',tag='status')
#---load the yaml specifications file
specs = self.load_specs()
#### status('done loading specs',tag='status')
#---read simulations from the slices dictionary
sns = specs['slices'].keys()
#---variables are passed directly to self.vars
self.vars = deepcopy(specs['variables']) if 'variables' in specs else {}
#---apply "+"-delimited internal references in the yaml file
for path,sub in [(i,j[-1]) for i,j in catalog(specs) if type(j)==list
and type(j)==str and re.match('^\+',j[-1])]:
source = delve(self.vars,*sub.strip('+').split('/'))
point = delve(specs,*path[:-1])
point[path[-1]][point[path[-1]].index(sub)] = source
for path,sub in [(i,j) for i,j in catalog(specs) if type(j)==str and re.match('^\+',j)]:
source = delve(self.vars,*sub.strip('+').split('/'))
point = delve(specs,*path[:-1])
point[path[-1]] = source
#---loop over all simulations to create groups and slices
self.save(quiet=True)
for route in [('slices',i) for i in sns]:
root,sn = delve(specs,*route),route[-1]
#---create groups
if 'groups' in root:
for group,select in root['groups'].items():
kwargs = {'group':group,'select':select,'sn':sn}
self.create_group(**kwargs)
root.pop('groups')
#---slice the trajectory
if 'slices' in root:
for sl,details in root['slices'].items():
#---! use a default group here?
for group in details['groups']:
kwargs = {'sn':sn,'start':details['start'],
'end':details['end'],'skip':details['skip'],'slice_name':sl}
kwargs['group'] = group
if 'pbc' in details: kwargs['pbc'] = details['pbc']
self.create_slice(**kwargs)
root.pop('slices')
if root != {}: raise Exception('[ERROR] unprocessed specifications %s'%str(root))
else: del root
#---we only save after writing all slices. if the slicer fails autoreload will find preexisting files
self.save(quiet=True)
checktime()
#---meta is passed to self.meta
if 'meta' in specs:
for sn in specs['meta']:
self.meta[sn] = specs['meta'][sn]
#---collections are groups of simulations
if 'collections' in specs: self.vars['collections'] = specs['collections']
#---calculations are executed last and organized in this loop
if 'calculations' in specs:
status('starting calculations',tag='status')
#---note that most variables including calc mirror the specs file
self.calc = dict(specs['calculations'])
#---infer the correct order for the calculation keys from their upstream dependencies
upstream_catalog = [i for i,j in catalog(self.calc) if 'upstream' in i]
#---if there are no specs required to get the upstream data object the user can either
#---...use none/None as a placeholder or use the name as the key as in "upstream: name"
for uu,uc in enumerate(upstream_catalog):
if uc[-1]=='upstream': upstream_catalog[uu] = upstream_catalog[uu]+[delve(self.calc,*uc)]
depends = {t[0]:[t[ii+1] for ii,i in enumerate(t) if ii<len(t)-1 and t[ii]=='upstream']
for t in upstream_catalog}
calckeys = [i for i in self.calc if i not in depends]
#---check that the calckeys has enough elements
list(set(calckeys+[i for j in depends.values() for i in j]))
#---! come back to this!
while any(depends):
ii,i = depends.popitem()
if all([j in calckeys for j in i]) and i!=[]: calckeys.append(ii)
else: depends[ii] = i
#---if a specific calculation name is given then only perform that calculation
if not calculation_name is None: calckeys = [calculation_name]
for calcname in calckeys:
details = specs['calculations'][calcname]
status('checking calculation %s'%calcname,tag='status')
new_calcs = self.interpret_specs(details)
#---perform calculations
for calc in new_calcs:
#---find the script with the funtion
fns = []
for (dirpath, dirnames, filenames) in os.walk('./'):
fns.extend([dirpath+'/'+fn for fn in filenames])
search = filter(lambda x:re.match('^\.\/[^ate].+\/%s\.py$'%calcname,x),fns)
if len(search)==0: raise Exception('\n[ERROR] cannot find %s.py'%calcname)
elif len(search)>1: raise Exception('\n[ERROR] redundant matches: %s'%str(search))
else:
sys.path.insert(0,os.path.dirname(search[0]))
function = unpacker(search[0],calcname)
status('computing %s'%calcname,tag='loop')
computer(function,calc=calc,workspace=self)
self.save()
checktime()
self.save()
import matplotlib
matplotlib.use("Agg")
if building_docs:
sys.path.insert(0, "../../../omni")
from base.workspace import Workspace
from base.tools import status, unpacker, flatten, unique
from base.store import picturedat, picturefind, datmerge
from base.timer import checktime
from functools import wraps
conf_paths, conf_gromacs = "paths.yaml", "gromacs.py"
# ---get the active workspace
if "work" not in globals() and not building_docs:
workspace = unpacker(conf_paths)["workspace_spot"]
work = Workspace(workspace, previous=False)
# ---INTERFACE
def workspace(func):
"""
This decorator tacks on the workspace to the function.
"""
@wraps(func)
def mod(*args, **kwargs):
kwargs["workspace"] = work
return func(*args, **kwargs)
