def load(): """Standard load sequence.""" global vmdmake, data, sns, calc, tempdir plotname = 'protein_rmsd' try: import vmdmake except: vmdmake_spot = os.path.join('calcs', 'vmdmake') if os.path.isdir(vmdmake_spot): raise Exception('could not import vmdmake but %s exists' % vmdmake_spot) else: bash('git clone http://github.com/bradleyrp/amx-vmd vmdmake', cwd='calcs') if 'data' not in globals(): data, calc = plotload(plotname) drop_dn = 'vmdmake_videos' #---store the snapshots in the post_plot_spot according to the tag tempdir = os.path.join(work.paths['post_plot_spot'], drop_dn) if not os.path.isdir(tempdir): os.mkdir(tempdir) status( 'snapshots are dropping to %s (delete them if you want to re-make them)' % tempdir, tag='note') do_video, do_smooth = True, True sns = work.sns()
def setup_virtualenv(self, sandbox=False): """ Create a virtualenvironment. """ def virtualenv_fail(name, extra=None): """ When virtualenv requires a system package we tell users that anaconda may be an option. """ message = 'failed to create a virtual environment: missing %s. ' % name if extra: message += extra raise Exception(message) if is_terminal_command('virtualenv'): virtualenv_fail('virtualenv') #---! automatically start redis? prevent dump.rdb? #---preliminary checks #---! default redis conf? #---! this needs to be somewhere else ??? the whole point of this is to avoid that. if False: if is_terminal_command('redis-cli'): virtualenv_fail( 'redis-cli', extra='if you have redis installed, ' + 'you can run "sudo /usr/sbin/redis-server --daemonize yes". ' + 'if your redis is local (via e.g. anacodna) you can omit "sudo"' ) #---you can sandbox or not venv_opts = "--no-site-packages " if sandbox else "--system-site-packages " #---note that virtualenv can re-run on the env folder without issue bash('virtualenv %senv' % venv_opts, log='logs/log-virtualenv')
def __init__(self, ): """ Create a factory environment from instructions in the config, and setup or refresh if necessary. """ #---always check that docker is running #---! can we also get some useful information from this check? bash('docker ps')
def setup_anaconda_refresh(self): """ Refresh the virtualenvironment. """ if self.use_python2: #! note that anaconda may have deprecated use of env/envs/py2/bin/activate self.loader_commands['env_activate'] = 'env/bin/activate py2' self.source_cmd = 'source env/bin/activate py2' #---we consult a conda YAML file and a PIP text list to install packages #---default values are built into the class above but they can be overridden config = read_config() reqs_conda = config.get('reqs_conda', self.reqs_conda) reqs_pip = config.get('reqs_pip', self.reqs_pip) if type(reqs_conda) != list: reqs_conda = [reqs_conda] if type(reqs_pip) != list: reqs_pip = [reqs_pip] #---install from the conda requirements list followed by pip (for packages not available on conda) for fn in reqs_conda: print('[STATUS] installing packages via conda from %s' % fn) #---we tell conda to ignore local user site-packages because version errors bash(self.source_cmd + ' && conda env update --file %s' % fn, log='logs/log-anaconda-conda-%s' % os.path.basename(fn)) for fn in reqs_pip: print('[STATUS] installing packages via pip from %s' % fn) bash(self.source_cmd + ' && pip install -r %s' % fn, log='logs/log-anaconda-conda-%s' % os.path.basename(fn))
def prepare(): """ Run once to prepare for videos. """ global vmdmake, tempdir try: import vmdmake except: vmdmake_spot = os.path.join('calcs', 'vmdmake') if os.path.isdir(vmdmake_spot): raise Exception('could not import vmdmake but %s exists' % vmdmake_spot) else: bash('git clone http://github.com/bradleyrp/amx-vmd vmdmake', cwd='calcs') if 'data' not in globals(): global data, calc data, calc = plotload(plotname) drop_dn = 'vmdmake_videos' #---store the snapshots in the post_plot_spot according to the tag tempdir = os.path.join(work.paths['post_plot_spot'], drop_dn) if not os.path.isdir(tempdir): os.mkdir(tempdir) status( 'snapshots are dropping to temporary folders in %s. it is your responsibility to delete them.' % tempdir + ' after the video is created we will not remake it unless you delete it.', tag='note') do_video, do_smooth = True, True
def show_running_factories(): """ Show all factory processes. Note that jupyter notebooks cannot be easily killed manually so use the full path and try `pkill -f`. Otherwise this function can help you clean up redundant factories by username. """ cmd = 'ps xao pid,user,command | egrep "([c]luster|[m]anage.py|[m]od_wsgi-express|[j]upyter)"' try: bash(cmd) except: print('[NOTE] no processes found using bash command: `%s`'%cmd)
def daemon_ender(fn,cleanup=True): """ Read a lock file and end the job with a particular message """ try: bash('bash %s'%fn) except Exception as e: print('[WARNING] failed to shutdown lock file %s with exception:\n%s'%(fn,e)) if cleanup: print('[STATUS] daemon successfully shutdown via %s'%fn) os.remove(fn)
def render_from_json(): """ Read video_catalog.json and make videos from it if they do not already exist. Video viles are written back to video_catalog.json. """ #---import settings due to problems with with open('video_requests.json') as fp: film_cuts = json.loads(fp.read()) vids = dict([(sn, {}) for sn in sns]) for cut_name, cut_spec in film_cuts.items(): #---loop over simulations for sn in sns: #---settings slice_path = calc['extras'][sn]['slice_path'] gro, xtc = [ os.path.join(work.postdir, '%s.%s' % (slice_path, j)) for j in ['gro', 'xtc'] ] tpr = work.raw.get_last(sn, subtype='tpr') view = vmdmake.VMDWrap(site=tempdir, gro=gro, xtc=xtc, tpr=tpr, frames='', xres=4000, yres=4000, **cut_spec.get('kwargs', {})) view.do('load_dynamic', 'standard', *cut_spec.get('does', [])) for sel in cut_spec.get('selections', []): view.select(**sel) view.do('reset', 'xview') view.command('scale by 1.2') view.video() view.command('animate goto last') view.command('') view['snapshot_filename'] = 'snap.%s.%s' % (cut_name, sn) view.do('snapshot') #---no repeats #---! currently the video suffix is hard-coded vid_fn = os.path.join('vid.%s.%s.%s' % (cut_name, sn, 'mp4')) is_complete = os.path.isfile(os.path.join(tempdir, vid_fn)) if not is_complete: view.show(quit=True) view.render(name='vid.%s.%s' % (cut_name, sn)) vid_fn_mp4 = os.path.join(tempdir, vid_fn) #---intervene to convert this to webm vid_fn = re.sub('\.mp4$', '.webm', vid_fn_mp4) if not is_complete: bash('ffmpeg -i %s -b:v 0 -crf 20 %s' % (vid_fn_mp4, vid_fn)) #---save the video file and cut details #---! should you resave more cut information? vids[sn][cut_name] = vid_fn #---files go back out to the interactive python with open('video_catalog.json', 'w') as fp: json.dump(vids, fp)
def clone_vmdmake(): """Clone a copy of VMDMAKE.""" from config import bash vmdmake_dn = 'calcs/codes/vmdmake' vmdmake_source = 'https://github.com/bradleyrp/amx-vmd' try: from codes import vmdmake except: if os.path.isdir(vmdmake_dn): raise Exception('cannot import vmdmake from codes but %s exists' % vmdmake_dn) bash('git clone %s %s' % (vmdmake_source, vmdmake_dn)) from codes import vmdmake globals()['vmdmake'] = vmdmake
def clone_calcs(source): """ Clone a calculations repository. """ config = read_config() if 'calculations_repo' in config and not os.path.isdir('calcs/.git'): raise Exception('config has a calculations repo registered but we cannot find calcs/.git') elif not 'calculations_repo' in config and os.path.isdir('calcs/.git'): raise Exception('found calcs/.git but no calculations_repo in the config') elif 'calculations_repo' in config and os.path.isdir('calcs/.git'): raise Exception('you already have a calculations repo at calcs') #---clone and register bash('git clone %s calcs'%source) config['calculations_repo'] = source write_config(config)
def setup_virtualenv_refresh(self): """ Refresh the virtualenvironment. """ for fn in self.reqs: print('[STATUS] installing packages via pip from %s' % fn) bash(self.source_cmd + ' && pip install -r %s' % fn, log='logs/log-virtualenv-pip-%s' % os.path.basename(fn)) #---custom handling for required upgrades #---! would be useful to make this systematic required_upgrades = [ 'Sphinx>=1.4.4', 'numpydoc', 'sphinx-better-theme', 'beautifulsoup4' ] #---! here and above we should source the environment first, but this is chicken-egg bash(self.source_cmd + ' && pip install -U %s' % ' '.join(["'%s'" % i for i in required_upgrades]), log='logs/log-virtualenv-pip')
def setup_anaconda(self): """ Set up anaconda. """ anaconda_location = self.config.get('anaconda_location', None) if not anaconda_location: raise Exception( 'download anaconda and run `make set anaconda_location <path>`' ) install_fn = abspath(anaconda_location) if not os.path.isfile(install_fn) and not ( os.path.islink(install_fn) and not os.path.isdir(install_fn)): raise Exception( 'cannot find %s. make sure you have a copy of anaconda there.' % install_fn + 'or run `make set anaconda_location=~/path/to/Miniconda3-latest-<architecture>.sh` ' 'to use a different path.') bash('bash %s -b -p %s/env' % (install_fn, os.getcwd()), log='logs/log-anaconda-env') if self.use_python2: #---we have to source the root anaconda here. later the source_cmd will refer to "py2" bash( ' && '.join([ 'source env/bin/activate', #---! very annoying anaconda errors so we update here on installation 'conda update -y conda', 'conda create python=2 -y -n py2' ]), log='logs/log-anaconda-env-py2') #---we use the conda environment handler to avoid using the user site-packages in ~/.local env_etc = 'env/envs/py2/etc' env_etc_conda = 'env/envs/py2/etc/conda' for dn in [env_etc, env_etc_conda]: if not os.path.isdir(dn): os.mkdir(dn) for dn in ['activate.d', 'deactivate.d']: os.mkdir(os.path.join(env_etc_conda, dn)) with open(os.path.join(env_etc_conda, 'activate.d', 'env_vars.sh'), 'w') as fp: fp.write('#!/bin/sh\nexport PYTHONNOUSERSITE=True\n') with open( os.path.join(env_etc_conda, 'deactivate.d', 'env_vars.sh'), 'w') as fp: fp.write('#!/bin/sh\nunset PYTHONNOUSERSITE\n')
def package_django_module(source,projname): """ Packages and installs a django module. Note that this is necessary for the initial connection, even if you use the development code. """ dev_dn = os.path.join(source,projname) pack_dn = os.path.join('pack',projname) if not os.path.isdir(dev_dn): raise Exception('cannot find %s'%dev_dn) if os.path.isdir(pack_dn): raise Exception('%s already exists'%pack_dn) #---copy the generic python packager shutil.copytree('mill/packer',pack_dn) #---copy the development code into the same directory #---! make this pythonic bash('cp -a %s %s'%(dev_dn,os.path.join(pack_dn,''))) find_and_replace(os.path.join('pack',projname,'setup.py'), ('^#---SETTINGS','packname,packages = \"%s\",[\"%s\"]'%(projname,projname))) find_and_replace(os.path.join('pack',projname,'MANIFEST.in'), ('APPNAME',projname)) #---prepare the package bash('python %s sdist'%os.path.join('pack',projname,'setup.py')) #---uninstall the package #try: bash('echo -e "y\n" | pip uninstall %s &> logs/log-pip-$projname'%projname) #except: pass #---install the package bash('pip install -U pack/%s/dist/%s-0.1.tar.gz'%(projname,projname),log='logs/log-pip-%s'%projname)
def confirm_env(): """ Make sure we are in the right conda environment. Obviously this will not work if you use the virtualenv. Factory setup adds activate_env pointer to the environment activator to config.py which is run every time you run a factory command. This is a helpful feature for one-off factory make commands because then the user doesn't have to worry about environments. It's annoying for development, so we source the environment ourselves, `make unset activate_env` and this check makes sure we don't make mistakes. """ hint = 'try `source env/bin/activate py2` or `make set activate_env="env/bin/activate py2"`' try: check = bash('conda info --json', catch=True) except: raise Exception( 'you are definitely in the wrong environment because I cannot find `conda`' + hint) conda_info = json.loads(check['stdout']) if not os.path.relpath(conda_info['default_prefix'], conda_info['conda_prefix']) == 'envs/py2': raise Exception('you are not in the right environment! ' + hint)
def setup_virtualenv(sandbox=False): """ MAIN SETUP ROUTINE """ config = read_config() if is_terminal_command('virtualenv'): virtualenv_fail('virtualenv') #---! automatically start redis? prevent dump.rdb? #---preliminary checks #---! default redis conf? if is_terminal_command('redis-cli'): virtualenv_fail( 'redis-cli', extra='if you have redis installed, ' + 'you can run "sudo /usr/sbin/redis-server --daemonize yes". ' + 'if your redis is local (via e.g. anacodna) you can omit "sudo"') venv_opts = "--no-site-packages " if sandbox else "--system-site-packages " #---note that virtualenv can re-run on the env folder without issue start_time = time.time() bash('virtualenv %senv' % venv_opts, log='logs/log-virtualenv') print('[STATUS] installing packages via pip...') bash( 'source env/bin/activate && pip install -r mill/requirements_virtualenv.txt', log='logs/log-virtualenv-pip') print('[NOTE] setup took %.1f minutes' % ((time.time() - start_time) / 60.)) required_upgrades = [ 'Sphinx>=1.4.4', 'numpydoc', 'sphinx-better-theme', 'beautifulsoup4' ] bash('pip install -U %s' % ' '.join(["'%s'" % i for i in required_upgrades]), log='logs/log-virtualenv-pip') #---record success config['setup_stamp'] = time.strftime('%Y%m%d%H%M%s') write_config(config)
def render_from_json(request_fn='video_requests.json', catalog_fn='video_catalog.json'): """ Read video_catalog.json and make videos from it if they do not already exist. Video viles are written back to video_catalog.json. """ sns = work.sns() #---import settings due to problems with if not os.path.isfile(request_fn): raise Exception('cannot find %s' % request_fn) with open(request_fn) as fp: film_cuts = json.loads(fp.read()) vids = dict([(sn, {}) for sn in sns]) for cut_name, cut_spec_loop in film_cuts.items(): #---loop over simulations for sn in sns: #---make a copy because we modify some arguments cut_spec = copy.deepcopy(cut_spec_loop) #---settings slice_path = calc['extras'][sn]['slice_path'] gro, xtc = [ os.path.join(work.postdir, '%s.%s' % (slice_path, j)) for j in ['gro', 'xtc'] ] #! legacy plot mode. remove this once omnicalc development branch is completed. #! ... it is fairly inefficient to parse the entire source just to get a single TPR #! ... so consider using meta to specify it explicitly, or some other method? try: tpr = work.raw.get_last(sn, subtype='tpr') except: work.parse_sources() tpr = work.source.get_last(sn, subtype='tpr') #! limit the number of frames if it is excessive. overridden by step in kwargs in cut_spec nframes_max = cut_spec.get('nframes_max', None) if nframes_max: nframes = ( (calc['extras'][sn]['end'] - calc['extras'][sn]['start']) / calc['extras'][sn]['skip']) # get the largest integer step size that will keep the number of frames below the max step = int(np.ceil(float(nframes) / nframes_max)) if step < 1: raise Exception('negative step size') else: step = 1 view = vmdmake.VMDWrap(site=tempdir, gro=gro, xtc=xtc, tpr=tpr, frames='', res=cut_spec.pop('resolution', (1000, 1000)), step=step, **cut_spec.get('kwargs', {})) view.do('load_dynamic', 'standard', *cut_spec.get('does', [])) for sel in cut_spec.get('selections', []): #! elaborate color handling here color_specific = sel.get('color_specific', None) color_specific_this = None if type(color_specific) == dict and set( color_specific.keys()) == {'eval'}: color_specific_this = eval(color_specific['eval']) sel['color_specific'] = True elif type(color_specific) in str_types: color_specific_this = str(color_specific) sel['color_specific'] = True if color_specific_this != None: view.set_color_cursor(color_specific_this) view.select(**sel) view.do('reset', 'xview') view.command('scale by %s' % cut_spec.get('zoom', 1.2)) if not cut_spec.get('debug', False): view.video() view.command('animate goto last') # render in higher detail view.command( 'display resize %d %d' % tuple(cut_spec.pop('resolution_snapshot', (4000, 4000)))) view['snapshot_filename'] = 'snap.%s.%s' % (cut_name, sn) view.do('snapshot') #---no repeats #---! currently the video suffix is hard-coded vid_fn = os.path.join('vid.%s.%s.%s' % (cut_name, sn, 'mp4')) is_complete = os.path.isfile(os.path.join(tempdir, vid_fn)) if not is_complete and not cut_spec.get('debug', False): view.show(quit=True) view.render(name='vid.%s.%s' % (cut_name, sn)) elif cut_spec.get('debug', False): status('debug mode runs view.show to make a snapshot') view.show(quit=True) else: status('video is already complete at %s' % os.path.join(tempdir, vid_fn)) if not cut_spec.get('debug', False): vid_fn_mp4 = os.path.join(tempdir, vid_fn) #---intervene to convert this to webm vid_fn = re.sub('\.mp4$', '.webm', vid_fn_mp4) if not is_complete: bash('ffmpeg -y -i %s -b:v 0 -crf 20 %s' % (vid_fn_mp4, vid_fn)) #---save the video file and cut details #---! should you resave more cut information? vids[sn][cut_name] = vid_fn #---files go back out to the interactive python with open(catalog_fn, 'w') as fp: json.dump(vids, fp)
def plot(): """Plot from the command line. See the film_cuts below.""" import json from config import bash try: import vmdmake except: #---clone vmdmake codes if they are absent vmdmake_spot = os.path.join('vmdmake') if os.path.isdir(vmdmake_spot): raise Exception('could not import vmdmake but %s exists' % vmdmake_spot) else: bash('git clone http://github.com/bradleyrp/amx-vmd vmdmake') #---clone the martini repo for the bonder code if not os.path.isdir('amx-martini'): bash('git clone http://github.com/bradleyrp/amx-martini') #---use a calculation to get the trajectory files, set by the martini_video_interactive entry in plots if 'data' not in globals(): data, calc = plotload(plotname) #---block: video requests drop_dn = 'vmdmake_videos' do_smooth = True lipid_material = ['goodsell', 'glass1', 'edgyglass', 'diffuse'][-1] film_cuts = { 'bilayer.side': { 'debug': False, 'zoom': 1.8, 'does': ['bonder'], 'nframes_max': 300, 'kwargs': { 'cgbondspath': os.path.join(os.getcwd(), 'amx-martini/bin/cg_bonds.tcl') }, 'selections': [{ 'lipids_r%d' % rnum: 'noh and resname %s' % resname, 'style': 'Licorice 0.3 12.0 12.0', 'smooth': do_smooth, lipid_material: True, 'color_specific': { 'eval': 'colorize(work.meta[sn],resname=\"' + resname + '\",named=True,overrides={"CHL1":"white"})' } } for rnum, resname in enumerate( work.vars['selectors']['resnames_lipid'] + ['CHL1'])] + [ dict(subject='protein and noh', style='Licorice 0.6 12.0 12.0', smooth=do_smooth, goodsell=True), dict(subject_cartoon='protein and noh', style='cartoon', diffuse=True, smooth=do_smooth, goodsell=True, color_specific='black'), ] }, } #---block: make videos #---store the snapshots in the post_plot_spot according to the tag tempdir = os.path.join(work.paths['post_plot_spot'], drop_dn) if not os.path.isdir(tempdir): os.mkdir(tempdir) status( 'snapshots are dropping to %s (delete them if you want to re-make them)' % tempdir, tag='note') sns = work.sns() #---! Jupyter has a bug currently forbidding Popen so we have to do this all day #---! mod from interactive with open('video_requests.json', 'w') as fp: json.dump(film_cuts, fp) render_from_json()
__doc__ = """ Automatic videomaker adapted for protein demo. This script and its companion were adapted from the protein version. """ #---block: imports if not is_live or not os.path.basename(os.getcwd())=='calcs': raise Exception('you must run this from jupyter in the factory') import json from config import bash try: import vmdmake except: #---clone vmdmake codes if they are absent vmdmake_spot = os.path.join('vmdmake') if os.path.isdir(vmdmake_spot): raise Exception('could not import vmdmake but %s exists'%vmdmake_spot) else: bash('git clone http://github.com/bradleyrp/amx-vmd vmdmake') #---clone the martini repo for the bonder code if not os.path.isdir('amx-martini'): bash('git clone http://github.com/bradleyrp/amx-martini') #---use a calculation to get the trajectory files, set by the martini_video_interactive entry in plots if 'data' not in globals(): data,calc = plotload(plotname) #---block: video requests for MARTINI bilayer simulations drop_dn = 'vmdmake_videos' do_smooth = True film_cuts = { 'bilayer.side':{'does':['bonder'],'kwargs':{'cgbondspath': os.path.join(os.getcwd(),'amx-martini/bin/cg_bonds.tcl')}, 'selections':[dict( lipids=' or '.join(['resname %s'%i for i in work.vars['selectors']['resnames_lipid']]), style='Licorice 1.000000 12.000000 12.000000',smooth=do_smooth,goodsell=True)]},}
def prepare_server(): """ Confirm that we are ready to serve. """ #---mod_wsgi is not available for conda on python 2 bash('LDLIBS=-lm pip install -U --no-cache-dir mod_wsgi')
def connect_single(connection_name,**specs): """ The big kahuna. Revamped recently. """ config = read_config() #---skip a connection if enabled is false if not specs.get('enable',True): return mkdir_or_report('data') mkdir_or_report('site') #---the site is equivalent to a django project #---the site draws on either prepackaged apps in the pack folder or the in-development versions in dev #---since the site has no additional data except that specified in connect.yaml, we can always remake it if os.path.isdir('site/'+connection_name): print("[STATUS] removing the site for \"%s\" to remake it"%connection_name) shutil.rmtree('site/'+connection_name) #---regex PROJECT_NAME to the connection names in the paths sub-dictionary #---note that "PROJECT_NAME" is therefore protected and always refers to the #---...top-level key in connect.yaml #---! note that you cannot use PROJECT_NAME in spots currently for key,val in specs.items(): if type(val)==str: specs[key] = re.sub('PROJECT_NAME',connection_name,val) elif type(val)==list: for ii,i in enumerate(val): val[ii] = re.sub('PROJECT_NAME',connection_name,i) #---paths defaults specs['plot_spot'] = specs.get('plot_spot',os.path.join('data',connection_name,'plot')) specs['post_spot'] = specs.get('post_spot',os.path.join('data',connection_name,'post')) specs['simulations_spot'] = specs.get('simulations_spot',os.path.join('data',connection_name,'sims')) specs['coords_spot'] = specs.get('coords_spot',os.path.join('data',connection_name,'coords')) #---intervene here to replace PROJECT_NAME in the string values of each spot for spotname,spot_details in specs.get('spots',{}).items(): for key,val in spot_details.items(): if type(val) in str_types: specs['spots'][spotname][key] = re.sub('PROJECT_NAME',connection_name,val) #---we also expand paths for route_to_data specs['spots'][spotname]['route_to_data'] = os.path.expanduser(os.path.abspath( specs['spots'][spotname]['route_to_data'])) #---cluster namer is set in a separate file cluster_namer = {} with open('mill/cluster_spec.py') as fp: exec(fp.read(),cluster_namer) for key in [i for i in cluster_namer if i not in cluster_namer['keepsakes']]: del cluster_namer[key] ###---DJANGO SETTINGS #---first define folders and (possibly) http git repos settings_custom = { 'SIMSPOT':abspath(specs['simulations_spot']), #---! hard-coded. get it from config.py?? 'AUTOMACS':'http://github.com/biophyscode/automacs', 'PLOT':abspath(specs['plot_spot']), 'POST':abspath(specs['post_spot']), 'COORDS':abspath(specs['coords_spot']), #---omnicalc locations are fixed 'CALC':abspath(os.path.join('calc',connection_name)), 'FACTORY':os.getcwd(), #---! get this from config.py 'CLUSTER':'cluster'} #---all paths are absolute unless they have a colon in them, in which case it is ssh or http #---we attach filesystem separators as well so that e.g. settings.SPOT can be added to relative paths settings_custom = dict([(key,os.path.join(os.path.abspath(val),'') if ':' not in val else val) for key,val in settings_custom.items()]) settings_custom['CLUSTER_NAMER'] = cluster_namer #---if the user does not supply a gromacs_config.py the default happens #---option to specify gromacs config file for automacs if 'gromacs_config' in specs: gromacs_config_fn = specs['gromacs_config'] if not os.path.isfile(gromacs_config_fn): raise Exception('cannot find gromacs_config file at %s'%gromacs_config_fn) settings_custom['GROMACS_CONFIG'] = os.path.join(os.getcwd(),gromacs_config_fn) else: settings_custom['GROMACS_CONFIG'] = False #---additional custom settings which are not paths #---if there is a public dictionary and we receive the "public" flag from make we serve public site if specs.get('public',None): site_port = specs['public'].get('port',8000) #---the notebook IP for django must be the public hostname, however in the get_public_ports function #---...we have an internal notebook_hostname for users who have a router if 'hostname' not in specs['public']: raise Exception('for public deployment you must add the hostname to the connection') #---the hostnames are a list passed to ALLOWED_HOSTS starting with localhost if type(specs['public']['hostname']) in str_types: hostnames = [specs['public']['hostname']] elif type(specs['public']['hostname'])==list: hostnames = specs['public']['hostname'] else: raise Exception('cannot parse hostname') hostnames.append('localhost') settings_custom['extra_allowed_hosts'] = list(set(hostnames)) #---the first hostname is the primary one settings_custom['NOTEBOOK_IP'] = hostnames[0] settings_custom['NOTEBOOK_PORT'] = specs['public'].get('notebook_port',site_port+1) #---serve locally else: #---note that notebook ports are always one higher than the site port site_port = specs.get('port',8000) settings_custom['NOTEBOOK_IP'] = 'localhost' settings_custom['NOTEBOOK_PORT'] = specs.get('port_notebook',site_port+1) settings_custom['extra_allowed_hosts'] = [] #---name this project settings_custom['NAME'] = connection_name ###---END DJANGO SETTINGS #---make local directories if they are absent or do nothing if the user points to existing data root_data_dir = 'data/'+connection_name #---always make data/PROJECT_NAME for the default simulation_spot therein mkdir_or_report(root_data_dir) for key in ['post_spot','plot_spot','simulations_spot']: mkdir_or_report(abspath(specs[key])) #---we always include a "sources" folder in the new simulation spot for storing input files mkdir_or_report(abspath(specs.get('coords_spot',os.path.join('data',connection_name,'coords')))) #---check if database exists and if so, don't make superuser make_superuser = not os.path.isfile(specs['database']) #---get automacs,omnicalc from a central place if it is empty automacs_upstream = specs.get('automacs',config.get('automacs',None)) msg = 'You can tell the factory where to get omnicalc/automacs by running e.g. '+\ '`make set automacs=http://github.com/someone/automacs`.' if not automacs_upstream: raise Exception('need automacs in config.py for factory or the connection. '+msg) #---! automacs_upstream is not being used? settings_custom['AUTOMACS'] = automacs_upstream automacs_branch = config.get('automacs_branch',None) if automacs_branch != None: settings_custom['AUTOMACS_BRANCH'] = automacs_branch omnicalc_upstream = specs.get('omnicalc',config.get('omnicalc',None)) if not omnicalc_upstream: raise Exception('need omnicalc in config.py for factory or the connection. '+msg) #---note that previous version of factory prepended a source command in front of every call #---...however the factory handles this for us now #---django is accessed via packages imported in settings.py which is why we have to package them #---...this saves us from making N copies of the development code #---! YOU NEED TO MAKE THE DEVELOPMENT POSSIBLE SOMEHWERE HEREABOUTS #---! hard-coding the location of the sources django_source = 'interface' #---! switching to new development codes...calculator not available yet for app in ['simulator','calculator']: if os.path.isdir('pack/%s'%app): shutil.rmtree('pack/%s'%app) #---always repackage! package_django_module(source=django_source,projname=app) #---one new django project per connection bash('django-admin startproject %s'%connection_name, log='logs/log-%s-startproject'%connection_name,cwd='site/') #---if the user specifies a database location we override it here if specs.get('database',None): database_path_change = "\nDATABASES['default']['NAME'] = '%s'"%( os.path.abspath(specs['database'])) else: database_path_change = '' #---all settings are handled by appending to the django-generated default #---we also add changes to django-default paths with open(os.path.join('site',connection_name,connection_name,'settings.py'),'a') as fp: fp.write(project_settings_addendum+database_path_change) #---only use the development code if the flag is set and we are not running public if specs.get('development',True) and not specs.get('public',False): fp.write('\n#---use the development copy of the code\n'+ 'import sys;sys.path.insert(0,os.path.join(os.getcwd(),"%s"))'%django_source) #---one more thing: custom settings specify static paths for local or public serve #if specs.get('public',None): # fp.write("\nSTATICFILES_DIRS = [os.path.join(BASE_DIR,'static')]") #else: fp.write("\nSTATICFILES_DIRS = [os.path.join('%s','interface','static')]"% os.path.abspath(os.getcwd())) #---write custom settings #---some settings are literals custom_literals = ['CLUSTER_NAMER'] with open(os.path.join('site',connection_name,connection_name,'custom_settings.py'),'w') as fp: #---! proper way to write python constants? fp.write('#---custom settings are auto-generated from mill.factory.connect_single\n') for key,val in settings_custom.items(): #---! is there a pythonic way to write a dictionary to a script of immutables if ((type(val) in str_types and re.match('^(False|True)$',val)) or key in custom_literals or type(val) in [bool,list,tuple]): out = '%s = %s\n'%(key,val) else: out = '%s = "%s"\n'%(key,val) fp.write(out) #---development uses live copy of static files in interface/static if not specs.get('public',None): #---link the static files to the development codes (could use copytree) os.symlink(os.path.join(os.getcwd(),django_source,'static'), os.path.join('site',connection_name,'static')) #---production collects all static files else: os.mkdir(os.path.join(os.getcwd(),'site',connection_name,'static_root')) bash('python manage.py collectstatic',cwd='site/%s'%connection_name) #---write project-level URLs with open(os.path.join('site',connection_name,connection_name,'urls.py'),'w') as fp: fp.write(project_urls) #---clone omnicalc if necessary omnicalc_previous = os.path.isdir('calc/%s'%connection_name) if not omnicalc_previous: bash('git clone %s calc/%s'%(omnicalc_upstream,connection_name), log='logs/log-%s-git-omni'%connection_name) #---if this is fresh we run `make setup` because that provides a minimal config.py bash('make setup',cwd=specs['calc']) else: print('[NOTE] found calc/%s'%connection_name) #---initial migration for all new projects to start the database #---...!!!!!!!!!!!!!! print('[NOTE] migrating ...') bash('python site/%s/manage.py makemigrations'%connection_name, log='logs/log-%s-migrate'%connection_name) bash('python site/%s/manage.py migrate --run-syncdb'%connection_name, log='logs/log-%s-migrate'%connection_name) print('[NOTE] migrating ... done') if make_superuser: print("[STATUS] making superuser") su_script = "from django.contrib.auth.models import User; "+\ "User.objects.create_superuser('admin','','admin');print;quit();" p = subprocess.Popen('python ./site/%s/manage.py shell'%(connection_name), stdin=subprocess.PIPE,stderr=subprocess.PIPE,stdout=open(os.devnull,'w'), shell=True,executable='/bin/bash') catch = p.communicate(input=su_script if sys.version_info<(3,0) else su_script.encode())[0] print("[STATUS] new project \"%s\" is stored at ./data/%s"%(connection_name,connection_name)) print("[STATUS] replace with a symlink if you wish to store the data elsewhere") #---now that site is ready we can write credentials if specs.get('public',None): #---write key,value pairs as Basic Auth user/passwords creds = specs['public'].get('credentials',{}) if creds: with open(os.path.join('site',connection_name,connection_name,'wsgi_auth.py'),'w') as fp: fp.write(code_check_passwd%str([(k,v) for k,v in creds.items()])) #---set up the calculations directory in omnicalc #---check if the repo pointer in the connection is a valid path new_calcs_repo = not (os.path.isdir(abspath(specs['repo'])) and ( os.path.isdir(abspath(specs['repo'])+'/.git') or os.path.isfile(abspath(specs['repo'])+'/HEAD'))) downstream_git_fn = os.path.join('calc',connection_name,'calcs','.git') #---if the repo key gives a web address and we already cloned it, then we do nothing and suggest a pull if ':' in specs['repo'] and os.path.isdir(downstream_git_fn): print('[NOTE] the calc repo (%s) appears to be remote and %s exists.'%( specs['calc'],downstream_git_fn)+'you should pull the code manually to update it') #---check that a calcs repo from the internet exists elif new_calcs_repo and re.match('^http',specs['repo']): #---see if the repo is a URL. code 200 means it exists if sys.version_info<(3,0): from urllib2 import urlopen else: from urllib.request import urlopen code = urlopen(specs['repo']).code if code!=200: raise Exception('repo appears to be http but it does not exist') else: bash('make clone_calcs source="%s"'%specs['repo'],cwd=specs['calc']) #---check that the repo has a colon in the path, implying a remote ssh connection is necessary elif new_calcs_repo and ':' in specs['repo']: print('[WARNING] assuming that the calcs repository is on a remote machine: %s'%specs['repo']) bash('make clone_calcs source="%s"'%specs['repo'],cwd=specs['calc']) #---if the calcs repo exists locally, we just clone it elif not new_calcs_repo and os.path.isdir(downstream_git_fn): print('[NOTE] git appears to exist at %s already and connection does not specify '% os.path.join(abspath(specs['repo']),'.git')+ 'an upstream calcs repo so we are continuing without action') elif not new_calcs_repo and not os.path.isfile(downstream_git_fn): bash('make clone_calcs source="%s"'%specs['repo'],cwd=specs['calc']) #---make a fresh calcs repo because the meta file points to nowhere else: os.mkdir(specs['repo']) bash('git init',cwd=specs['repo']) #---after making a blank repo we put a placeholder in the config bash('make set calculations_repo="no_upstream"',cwd=specs['calc']) #---also generate a blank metadata so that the interface works bash('make blank_meta make_template=False',cwd=specs['calc']) msg = ('When connecting to project %s, the "repo" flag in your connection file points to nowhere. ' 'We made a blank git repository at %s. You should develop your calculations there, push that ' 'repo somewhere safe, and distribute it to all your friends, who can use the "repo" flag to ' 'point to it when they start their factories.') print('\n'.join(['[NOTE] %s'%i for i in textwrap.wrap( msg%(connection_name,specs['repo']),width=60)])) #---pass a list of meta_filters through (can be list of strings which are paths or globs) calc_meta_filters = specs.get('calc_meta_filters',None) if calc_meta_filters: bash('make unset meta_filter',cwd=specs['calc']) for filt in calc_meta_filters: #---note that meta_filter is turned into a list in config.py in omnicalc bash('make set meta_filter="%s"'%filt,cwd=specs['calc']) #---configure omnicalc #---note that make set commands can change the configuration without a problem bash('make set post_data_spot=%s'%settings_custom['POST'],cwd=specs['calc']) bash('make set post_plot_spot=%s'%settings_custom['PLOT'],cwd=specs['calc']) #---! needs to interpret simulation_spot, add spots functionality #---! previously ran register_calculation.py here -- may be worth recapping in this version? #---! prepare vhost file here when it's ready #---??? IS THIS IT ??? #---write spots to config if 'spots' in specs: config_fn = os.path.join(specs['calc'],'config.py') with open(config_fn) as fp: config_omni = eval(fp.read()) config_omni['spots'] = specs['spots'] import pprint #---write the config with open(config_fn,'w') as fp: fp.write('#!/usr/bin/env python -B\n'+str(pprint.pformat(config_omni,width=110))) #---add the environment to omnicalc. this allows the publicly-served omnicalc to find the environment #---...when root is running it. it also means users do not have to remember to source the environment #---...when they are doing calculations "manually" from their project's omnicalc folder. note that there #---...is a slowdown if you are used to always sourcing the environment yourself, but you could just as #---...easily remove the flag from the config.py to recover the original behavior if 'activate_env' in config: env_path = "%s %s"%(os.path.join(os.path.abspath(config['activate_env'].split()[0])), config['activate_env'].split()[1]) bash('make set activate_env="%s"'%env_path,cwd=specs['calc'])
mvecs[:, fr], nprots_list, handle=handle, outdir=tmpdir, fi=fi, fn=handle + '.fr.%04d' % fi, pbc_expand=1.0, smooth=1., panelspecs=panelspecs, square=square_layout, extrema=extrema, fs=fs, titles=titles, cmap_mpl_name=cmap_mpl_name, ) for fi, fr in enumerate(frameset)) # print the film. you could add codec options here. the filter slows down the movie. ffmpegcmd = ['ffmpeg','-i',tmpdir+'/'+handle+'.fr.%04d.png','-b:v','0','-crf','20']+\ (['-filter:v','setpts=2.0*PTS'] if False else [])+[' '+outdir+'/'+handle+'.mp4'] print(' '.join(ffmpegcmd)) # ffmpeg is very particular status('calling ffmpeg via: %s' % ' '.join(ffmpegcmd), tag='bash') #! subprocess dislikes the list of arguments here for some reason try: bash(' '.join(ffmpegcmd), cwd=tmpdir) except Exception as e: raise Exception( 'failed with exception %s.' % e + '\nyou may need to adjust the ffmpeg call and render manually.') status('[STATUS] video rendered to ' + outdir + '/' + handle + '.mpeg')
fs=fs, titles=titles, cmap_mpl_name=cmap_mpl_name) sys.exit(1) joblib.Parallel(n_jobs=ppn, verbose=10)(joblib.delayed( print_birdseye_snapshot_render, joblib.pool.has_shareable_memory)( surfs[:, fr], protein_pts[fr], mvecs[:, fr], nprots_list, handle=handle, outdir=tmpdir, fi=fi, fn=handle + '.fr.%04d' % fi, pbc_expand=1.0, smooth=1., panelspecs=panelspecs, extrema=extrema, fs=fs, titles=titles, cmap_mpl_name=cmap_mpl_name, ) for fi, fr in enumerate(frameset)) #---print the film ffmpegcmd = ['ffmpeg','-i',tmpdir+'/'+handle+'.fr.%04d.png','-vcodec','mpeg2video','-q:v','0']+\ (['-filter:v','setpts=2.0*PTS'] if False else [])+[' '+outdir+'/'+handle+'.mpeg'] print(' '.join(ffmpegcmd)) #---! ffmpeg call is currently broken bash(ffmpegcmd, cwd=work.paths['post_plot_spot']) status('[STATUS] video rendered to ' + outdir + '/' + handle + '.mpeg')
nmol = len(m2i) #---note that depending on the PBC links we get a variable number of points_inside = np.array([ lipid_mesh['%d.%d.points' % (top_mono, fr)][:nmol] for fr in range(nframes) ]) windows = np.array([ np.arange(j, j + smooth_window) for j in np.arange(0, nframes - smooth_window) ]) points_inside_smooth = np.array( [points_inside[w].mean(axis=0) for w in windows]) #---render in parallel basic_compute_loop( compute_function=render_hydrogen_bonding_pattern, looper=[ dict(fr=fr, frameno=frameno) for frameno, fr in enumerate(valid_frames) ], run_parallel=True, debug=False) #---render when complete try: # https://superuser.com/questions/1005315/interpolation-with-ffmpeg cmd = 'ffmpeg -i "snap.%05d.v1.png" ' + 'mov.hydrogen_bonding_pattern.%s' % sn + '.mp4' bash(cmd, cwd=out_dn) except: status('failed to render the video. try "%s" in %s' % (cmd, out_dn)) del lipid_mesh