def test_LocalPath(self): """LocalPath test""" os.environ['PRJ_INTERFACES'] = os.path.expanduser('~') S = self S.path = [] S.l = LocalPath() ## Example 1; create from fragments S.l.set_fragments(('/home/Bis/johan', 'PRJ_INTERFACES'), ('/c11/com_wet/ref.com', None)) S.path += ['Example 1:\n %s : %s \n' % (S.l.formatted(), S.l.local())] S.assert_('johan' not in S.l.local()) ## Example 2; create from path with custom variable S.l.set_path('/home/Bis/raik/data/tb/interfaces/c11/com_wet/ref.com', USER='******') S.path += ['Example 2:\n %s : %s \n' % (S.l.formatted(), S.l.local())] S.assertEqual( S.l.formatted(),\ '{/home/Bis/raik|$USER}/data/tb/interfaces/c11/com_wet/ref.com' ) ## Example 3; create from non-existing path S.l.set_path('/home/xyz/data/tb/interfaces/c11/com_wet/ref.com') S.path += ['Example 3:\n %s : %s \n' % (S.l.formatted(), S.l.local())] S.assert_(S.l.formatted() == S.l.local()) ## Example 4; create from existing path with automatic substitution S.l.set_path(T.testRoot() + '/com') S.path += ['Example 4:\n %s : %s \n' % (S.l.formatted(), S.l.local())] S.assertEqual( S.l.formatted(), '{%s|$projectRoot}/Biskit/testdata/com' % T.projectRoot()) S.assertTrue(os.path.exists(S.l.local())) ## Example 5; rule out stray substitutions S.l.set_path(T.projectRoot() + '/tmp/com', maxSub=1, TMP='/tmp') S.path += ['Example 5:\n %s : %s \n' % (S.l.formatted(), S.l.local())] S.assertEqual(S.l.formatted(), '{%s|$projectRoot}/tmp/com' % T.projectRoot()) self.assertEqual(S.l.fragments[0][1], 'projectRoot') if S.local: for p in S.path: print p
def __init__(self, dat, chunk, hosts, outFolder, skipWat=0, amber=0, sort=0, **kw): """ @param dat: data dictionary @type dat: dict @param chunk: chunk size passed to slave @type chunk: int @param hosts: list of host-names @type hosts: [str] @param outFolder: alternative output folder @type outFolder: str """ niceness = {'default': 0} slave_script = T.projectRoot() + '/Biskit/StructureSlave.py' TrackingJobMaster.__init__(self, dat, chunk, hosts, niceness, slave_script, **kw) self.options = {} self.options['out'] = outFolder self.options['skipRes'] = None if skipWat: self.options['skipRes'] = ['WAT','TIP3','H2O','WWW','Na+','Cl-'] if kw.has_key('show_output'): self.options['report'] = not kw['show_output'] self.options['amber'] = amber self.options['sort'] = sort
def test_LocalPath( self ): """LocalPath test""" os.environ['PRJ_INTERFACES'] = os.path.expanduser('~') S = self S.path = [] S.l = LocalPath() ## Example 1; create from fragments S.l.set_fragments( ('/home/Bis/johan','PRJ_INTERFACES'), ('/c11/com_wet/ref.com', None) ) S.path += [ 'Example 1:\n %s : %s \n'%(S.l.formatted(), S.l.local()) ] S.assert_( 'johan' not in S.l.local() ) ## Example 2; create from path with custom variable S.l.set_path( '/home/Bis/raik/data/tb/interfaces/c11/com_wet/ref.com', USER='******' ) S.path += [ 'Example 2:\n %s : %s \n'%(S.l.formatted(), S.l.local()) ] S.assertEqual( S.l.formatted(),\ '{/home/Bis/raik|$USER}/data/tb/interfaces/c11/com_wet/ref.com' ) ## Example 3; create from non-existing path S.l.set_path( '/home/xyz/data/tb/interfaces/c11/com_wet/ref.com' ) S.path += [ 'Example 3:\n %s : %s \n'%(S.l.formatted(), S.l.local()) ] S.assert_( S.l.formatted() == S.l.local() ) ## Example 4; create from existing path with automatic substitution S.l.set_path( T.testRoot() + '/com' ) S.path += [ 'Example 4:\n %s : %s \n'%(S.l.formatted(), S.l.local()) ] S.assertEqual( S.l.formatted(), '{%s|$projectRoot}/Biskit/testdata/com' % T.projectRoot()) ## Example 5; rule out stray substitutions S.l.set_path( T.projectRoot() + '/tmp/com', maxSub=1, TMP='/tmp' ) S.path += [ 'Example 5:\n %s : %s \n'%(S.l.formatted(), S.l.local()) ] S.assertEqual( S.l.formatted(), '{%s|$projectRoot}/tmp/com'% T.projectRoot()) self.assertEqual( S.l.fragments[0][1], 'projectRoot' ) if S.local: for p in S.path: print p
def __init__(self, trajFiles, n_hosts=20, **kw): """ @param trajFiles: list of trajectory files @type trajFiles: [ str ] @param n_hosts: number of hosts to use @type n_hosts: int """ dat = {} i = 0 for f in trajFiles: dat[i] = T.absfile(f) i += 1 niceness = nice_dic hosts = nodes_all[:n_hosts] slave_script = T.projectRoot() + '/Biskit/QualSlave.py' TrackingJobMaster.__init__(self, dat, 1, hosts, niceness, slave_script, **kw)
def __init__(self, trajFiles, n_hosts=20, **kw): """ @param trajFiles: list of trajectory files @type trajFiles: [ str ] @param n_hosts: number of hosts to use @type n_hosts: int """ dat = {} i = 0 for f in trajFiles: dat[i] = T.absfile( f ) i += 1 niceness = nice_dic hosts = nodes_all[ :n_hosts ] slave_script = T.projectRoot() +'/Biskit/QualSlave.py' TrackingJobMaster.__init__(self, dat, 1, hosts, niceness, slave_script, **kw)
class Master(TrackingJobMaster): ## Slave script that goes with this master slave_script = projectRoot() + '/Biskit/PVM/ExampleSlave.py' def __init__(self, verbose=1, *args, **kw): """ Parameters nedded by master and/or slave. """ self.verbose = verbose TrackingJobMaster.__init__(self, verbose=verbose, *args, **kw) def getInitParameters(self, slave_tid): """ Hand over parameters to slave once. @param slave_tid: slave task id @type slave_tid: int @return: dictionary with init parameters @rtype: {param:value} """ return {'progress_str':'slave calculating..'} def cleanup( self ): """ Tidy up tasks. """ if self.verbose: print "Cleaning up..." def done( self ): """ Called when master is done. """ if self.verbose: print "Now we are done."
""" import os.path, copy import Biskit.oldnumeric as N0 import Biskit.tools as T import Biskit.settings as settings import Biskit.mathUtils as MU from Biskit.PVM.TrackingJobMaster import TrackingJobMaster from Biskit.PVM.hosts import cpus_all, nice_dic from Biskit import PDBModel, PDBProfiles, EHandler, StdLog from Biskit.Dock import Complex slave_path = T.projectRoot() + "/Biskit/AmberEntropySlave.py" class AmberEntropyMaster(TrackingJobMaster): """ Run many AmberEntropist calculations on many nodes. The Master has a standard set of 13 protocols to run on rec, lig, and com trajectories, as well as on every single member trajectory - in total 113. It accepts one variable parameter, e.g. s(tart). Each protocol is then run for all values of the variable parameter. A protocol is simply a set of options that are passed on to the AmberEntropist (which is run from within AmberEntropySlave). Comparing the different protocols allows to more or less separate random from real correlations, rigid body from intermolecular vibrations, etc.
class AlignerMaster(TrackingJobMaster): slave_script = projectRoot() + '/Biskit/Mod/AlignerSlave.py' def __init__(self, hosts, folders, pdbFolder=None, fastaTemplates=None, fastaSequences=None, fastaTarget=None, ferror=None, verbose=1, **kw): """ @param hosts: list of host-names @type hosts: [str] @param folders: list of project directories (path) @type folders: [str] @param pdbFolder: pdb directory (*.alpha for Aligner) @type pdbFolder: str @param fastaTemplates: path to find 'templates.fasta' @type fastaTemplates: str @param fastaSequences: path to find 'nr.fasta' @type fastaSequences: str @param fastaTarget: path to find 'target.fasta' @type fastaTarget: str @param ferror: filename to output errors from the Slave @type ferror: str @param verbose: verbosity level (default: 1) @type verbose: 1|0 @param kw: additional TrackingJobMaster arguments:: chunk_size - int, number of items that are processed per job niceness - {str_host-name: int_niceness} slave_script - str, absolute path to slave-script show_output - 1|0, display one xterm per slave [0] add_hosts - 1|0, add hosts to PVM before starting [1] @type kw: param=value """ self.verbose = verbose self.folders = folders self.pdbFolder = pdbFolder self.fastaTemplates = fastaTemplates self.fastaSequences = fastaSequences self.fastaTarget = fastaTarget self.ferror = ferror or 'AlignSlaveErrors.out' data = self.setupJobs() TrackingJobMaster.__init__(self, data=data, chunk_size=1, hosts=hosts, slave_script=self.slave_script, redistribute=0, verbose=verbose, **kw) def __dir_or_none(self, folder, filename): if filename is None: return None return os.path.join(folder, filename) def setupJobs(self): """ Prepare the job dictionnary for 'AlignerSlave' @return: input informations for aligner for each project @rtype: {{str}} """ r = {} for f in self.folders: aligner_input = {} aligner_input['outFolder'] = T.absfile(f) aligner_input['fastaTemplates'] = \ self.__dir_or_none( f, self.fastaTemplates ) aligner_input['fastaSequences'] = \ self.__dir_or_none( f, self.fastaSequences ) aligner_input['fastaTarget'] = \ self.__dir_or_none( f, self.fastaTarget ) pdb_list = [] if self.pdbFolder is None: pdb_list = None else: pdbfiles = os.listdir(f + self.pdbFolder) for pdb in pdbfiles: pdb_list.append(f + self.pdbFolder + '/%s' % pdb) aligner_input['pdbFiles'] = pdb_list r[T.absfile(f)] = aligner_input return r def getInitParameters(self, slave_tid): """ Hand over parameters to slave once. @param slave_tid: slave task id @type slave_tid: int @return: dictionary with init parameters @rtype: {param:value} """ return { 'progress_str': 'slave calculating..', 'ferror': self.ferror, 'os.environ': os.environ } def cleanup(self): print "Cleaning up..." def done(self): print "Done aligning."
class ModelMaster(TrackingJobMaster): slave_script = projectRoot() + '/Biskit/Mod/ModelSlave.py' def __init__(self, hosts, folders, fastaTarget=None, f_pir=None, template_folder=None, fout=None, starting_model=1, ending_model=10, ferror=None, verbose=1, **kw): """ @param hosts: list of host-names @type hosts: [str] @param folders: list of project directories (path) @type folders: [str] @param fastaTarget: path to find 'target.fasta' @type fastaTarget: str @param f_pir: alignment filename @type f_pir: str @param template_folder: directories for input atom files @type template_folder: str @param fout: default modeller input file 'modeller.top' @type fout: str @param starting_model: index of the first model (default: 1) @type starting_model: int @param ending_model: index of the last model (default: 10) @type ending_model: int @param ferror: filename to output errors from the Slave @type ferror: str @param verbose: verbosity level (default: 1) @type verbose: 1|0 @param kw: additional TrackingJobMaster arguments:: chunk_size - int, number of items that are processed per job niceness - {str_host-name: int_niceness} slave_script - str, absolute path to slave-script show_output - 1|0, display one xterm per slave [0] add_hosts - 1|0, add hosts to PVM before starting [1] @type kw: param=value """ self.verbose = verbose self.folders = folders self.fastaTarget = fastaTarget self.f_pir = f_pir self.template_folder = template_folder self.fout = fout self.starting_model = starting_model self.ending_model = ending_model self.ferror = ferror or 'ModelSlaveErrors.out' data = self.setupJobs() TrackingJobMaster.__init__(self, data=data, chunk_size=1, hosts=hosts, slave_script=self.slave_script, redistribute=0, verbose=verbose, **kw) def __dir_or_none(self, folder, filename): if filename is None: return None return os.path.join(folder, filename) def setupJobs(self): """ Prepare the job dictionnary for 'ModelSlave' @return: input informations for modeller for each project @rtype: {{str}} """ r = {} for f in self.folders: modeller_input = {} modeller_input['outFolder'] = T.absfile(f) modeller_input['fastaTarget'] = \ self.__dir_or_none( f, self.fastaTarget ) modeller_input['f_pir'] = self.__dir_or_none(f, self.f_pir) modeller_input['template_folder'] = \ self.__dir_or_none( f, self.template_folder ) modeller_input['starting_model'] = self.starting_model modeller_input['ending_model'] = self.ending_model r[T.absfile(f)] = modeller_input return r def getInitParameters(self, slave_tid): """ Hand over parameters to slave once. @param slave_tid: slave task id @type slave_tid: int @return: dictionary with init parameters @rtype: {param:value} """ return {'progress_str': 'slave calculating..', 'ferror': self.ferror} def cleanup(self): if self.verbose: print "Cleaning up..." def done(self): if self.verbose: print "Done modeling."
class TrajFlexMaster(TrackingJobMaster): """ Parallize calculation of pairwise rmsd between the frames of one or two trajectories. """ slave_script = T.projectRoot() + '/Biskit/TrajFlexSlave.py' def __init__(self, traj1, traj2=None, hosts=hosts.cpus_all, niceness=hosts.nice_dic, show_output=0, add_hosts=0, log=None, slaveLog=None, verbose=1, only_off_diagonal=1, only_cross_member=0): """ @param traj1: Trajectory or EnsembleTraj, traj1 and 2 must have the same atom content. If only traj1 is given, the pairwise rms is calculated between its frames. @type traj1: Trajectory OR EnsembleTraj @param traj2: see traj1 @type traj2: Trajectory OR EnsembleTraj ## @param aMask: atom mask, consider only subset of atoms (default: all) ## @type aMask: [0|1] @param hosts: slave hosts to be used (default: L{Biskit.PVM.hosts.cpus_all}) @type hosts: [str] @param niceness: { str:int, 'default':int }, nice value for each host @type niceness: dict @param show_output: open xterm window for each slave (default: 0) @type show_output: 0|1 @param add_hosts: add hosts to PVM before starting (default: 0) @type add_hosts: 1|0 @param log: log file for master (default: None-> StdErr ) @type log: LogFile @param slaveLog: slave log (default: None->'TrajFlexSlave_errors.log') @type slaveLog: LogFile @param verbose: print progress infos (default: 1) @type verbose: 0|1 @param only_off_diagonal: Don't calculate self-rms of frames. Only considered for a single trajectory (default: 1) @type only_off_diagonal: 0|1 @param only_cross_member: Don't calculate rms between frames from same member trajectory only considered for a single trajectory(requires EnsembleTraj) (default: 0) @type only_cross_member: 0|1 """ ## create temporary folder accessible to all slaves self.outFolder = tempfile.mktemp('trajFlex_', dir=settings.tempDirShared) os.mkdir(self.outFolder) self.log = log or ErrLog() self.slaveLog = slaveLog or LogFile('TrajFlexSlave_errors.log') self.verbose = verbose self.hosts = hosts self.traj_1 = traj1 self.traj_2 = traj2 self.only_off_diagonal = traj2 is None and only_off_diagonal self.only_cross_member = traj2 is None and only_cross_member self.trajMap = None if traj2 is None: self.trajMap = self.memberMap(traj1) ## pickle chunks of coordinate frames frame_files_1 = self.__dumpFrames(traj1, self.outFolder, 'traj1') ## None if traj2 is None frame_files_2 = self.__dumpFrames(traj2, self.outFolder, 'traj2') ## assemble job dict self.tasks = self.__taskDict(frame_files_1, frame_files_2) chunk_size = 1 TrackingJobMaster.__init__(self, self.tasks, chunk_size, hosts, niceness, self.slave_script, show_output=show_output, verbose=verbose, add_hosts=add_hosts) def getInitParameters(self, slave_tid): """ @param slave_tid: slave task id @type slave_tid: int @return: dictionary with init parameters @rtype: {param:value} """ return { 'ferror': self.slaveLog.fname, 'trajMap': self.trajMap, 'only_off_diagonal': self.only_off_diagonal, 'only_cross_member': self.only_cross_member } def __windowSize(self, n_per_node, n_nodes, n_frames): """ @param n_per_node: how many chunks should be generated per node @type n_per_node: int @param n_nodes: number of slave nodes @type n_nodes: int @param n_frames: length of trajectory @type n_frames: int @return: calculate number of frames per chunk @rtype: int """ r = int(round(n_frames * 1.0 / N0.sqrt(n_per_node * n_nodes))) if r > 25: return r return 25 def cleanup(self): """ Tidy up. """ T.tryRemove(self.outFolder, verbose=self.verbose, tree=1) def __getFrameWindows(self, traj, n_frames): """ Divide frame indices into chunks. @param n_frames: number of frames per chunk @type n_frames: int @return: list with start and stop frame index of each chunk @rtype: [(int, int)] """ if traj is None: return None l = len(traj) ## number of windows n, rest = l / n_frames, l % n_frames if rest: n += 1 ## get start and end frame for all windows i_windows = [] for i in range(0, n): start, stop = i * n_frames, i * n_frames + n_frames if stop > l: stop = l if i == n - 1: stop = l i_windows.append((start, stop)) return i_windows def __taskDict(self, f_frames_1, f_frames_2): """ @param f_frames_1: file name of chunk 1 of frames @type f_frames_1: {(int, int) : str} @param f_frames_2: file name of chunk 2 of frames @type f_frames_2: {(int, int) : str} @return: { ((int, int),(int, int)) : (str, str) } @rtype: {((int, int),(int, int)) : (str, str)} """ intra_traj = f_frames_2 is None if intra_traj: if self.verbose: self.log.add('Intra-trajectory calculation requested.') f_frames_2 = f_frames_1 if self.verbose: self.log.write('setting up task list...') i_windows = f_frames_1.keys() j_windows = f_frames_2.keys() ## assemble dict with all combinations of windows tasks = {} for i in range(len(i_windows)): for j in range(i * intra_traj, len(j_windows)): start_i, stop_i = i_windows[i] start_j, stop_j = j_windows[j] key = ((i_windows[i], j_windows[j])) tasks[key] = (f_frames_1[i_windows[i]], f_frames_2[j_windows[j]]) if self.verbose: self.log.add('done') return tasks def __dumpFrames(self, traj, outFolder, prefix): """ @param traj: Trajectory @type traj: Trajectory @param outFolder: folder for pickled arrays @type outFolder: str @param prefix: file name prefix @type prefix: str @return: { (int,int) : str } OR None, if traj is None @rtype: {(int,int) : str} """ if traj is None: return None if self.verbose: self.log.write('dumping frame chunks...') n_frames = self.__windowSize(20, len(self.hosts), len(traj)) i_windows = self.__getFrameWindows(traj, n_frames) r = {} for i in range(len(i_windows)): w = i_windows[i] a = traj.frames[w[0]:w[1]] f = outFolder + '/%s_%i_to_%i.dat' % ((prefix, ) + w) T.dump(a, f) r[w] = f if self.verbose and i % (len(i_windows) / 50 + 1) == 0: self.log.write('#') if self.verbose: self.log.add('done') return r def memberMap(self, traj): """ Tell which traj frame belongs to which member trajectory. @param traj: Trajectory @type traj: Trajectory @return: member index of each frame OR None if traj is not a EnsembleTraj @rtype: [ int ] OR None """ if not isinstance(traj, EnsembleTraj): return None r = N0.zeros(len(traj), N0.Int) for i in range(traj.n_members): mi = traj.memberIndices(i) N0.put(r, mi, i) return r.tolist() def getResult(self, mirror=0): """ Get result matrix ordered such as input trajectory. @param mirror: mirror the matrix at diagonal (default: 1) (only for intra-traj) @type mirror: 1|0 @return: array( (n_frames, n_frames), 'f'), matrix of pairwise rms @rtype: array """ if self.verbose: self.log.write('assembling result matrix...') intra_traj = self.traj_2 is None n1 = n2 = len(self.traj_1) if self.traj_2 is not None: n2 = len(self.traj_2) a = N0.zeros((n1, n2), N0.Float32) if self.verbose: self.log.write('#') for key, value in self.result.items(): i_start, i_stop = key[0] j_start, j_stop = key[1] window = N0.reshape(value, (i_stop - i_start, j_stop - j_start)) window = window.astype(N0.Float32) a[i_start:i_stop, j_start:j_stop] = window if self.verbose: self.log.write('#') if intra_traj: for i in range(N0.shape(a)[0]): for j in range(i, N0.shape(a)[1]): if a[j, i] == 0: a[j, i] = a[i, j] else: a[i, j] = a[j, i] if self.verbose: self.log.write('#') if intra_traj and not mirror: for i in range(N0.shape(a)[0]): for j in range(i, N0.shape(a)[1]): a[j, i] = 0. if self.verbose: self.log.add('done') return a def rmsMatrixByMember(self, mirror=0, step=1): """ Get result matrix ordered first by member then by time. (requires EnsembleTraj) @param mirror: mirror matrix at diagonal (only for intra-traj. rms) (default: 0) @type mirror: 0|1 @param step: take only every step frame [1] @type step: int """ intra_traj = self.traj_2 is None m = self.getResult(mirror=intra_traj) i1 = i2 = self.traj_1.argsortMember(step=step) if self.traj_2 is not None: i2 = self.traj_2.argsortMember(step=step) a = N0.take(m, i1, 0) a = N0.take(a, i2, 1) if intra_traj and not mirror: for i in range(N0.shape(a)[0]): for j in range(i, N0.shape(a)[1]): a[j, i] = 0. return a def rmsList(self): """ @return: list of all calculated pairwise rms values @rtype: [float] @raise FlexError: if there are no results yet """ r = [] for v in self.result.values(): r.extend(v) if not r: raise FlexError, "No results yet." return r def averageRms(self): """ @return: average pairwise rmsd and it's standard deviation @rtype: (float, float) @raise FlexError: if there are no results yet """ r = self.rmsList() return N0.average(r), mathUtils.SD(r)
""" import os.path, copy import numpy.oldnumeric as N import Biskit.tools as T import Biskit.settings as settings import Biskit.mathUtils as MU from Biskit.PVM.TrackingJobMaster import TrackingJobMaster from Biskit.PVM.hosts import cpus_all, nice_dic from Biskit import PDBModel, PDBProfiles, EHandler, StdLog from Biskit.Dock import Complex slave_path = T.projectRoot()+"/Biskit/AmberEntropySlave.py" class AmberEntropyMaster(TrackingJobMaster): """ Run many AmberEntropist calculations on many nodes. The Master has a standard set of 13 protocols to run on rec, lig, and com trajectories, as well as on every single member trajectory - in total 113. It accepts one variable parameter, e.g. s(tart). Each protocol is then run for all values of the variable parameter. A protocol is simply a set of options that are passed on to the AmberEntropist (which is run from within AmberEntropySlave). Comparing the different protocols allows to more or less separate random from real correlations, rigid body from intermolecular vibrations, etc. Results are put into a tree-shaped dictionary of dictionaries. The
## BISKIT_PATH = T.projectRoot() ## Hack to make test suite path independent try: m = M.SettingsManager(__CFG_DEFAULT, __CFG_USER, createmissing=True ) m.updateNamespace( locals() ) except Exception, why: B.EHandler.fatal( 'Error importing Biskit settings') ## ## Create some settings on the fly ## python_bin = sys.executable projectRoot= T.projectRoot() pymol_scripts = T.dataRoot() + '/pymol/' ################################### ## required environment variables. ## format: ENV_NAME : path_example ## Todo: These need to go to their Exe_*.dat files env = {} blast_env = {'BLASTDB':'/home/Bis/raik/data/prog/blast/db', 'BLASTMA':'/home/Bis/johan/APPLICATIONS/blast'}
## BISKIT_PATH = T.projectRoot() ## Hack to make test suite path independent try: m = M.SettingsManager(__CFG_DEFAULT, __CFG_USER, createmissing=True) m.updateNamespace(locals()) except Exception, why: B.EHandler.fatal('Error importing Biskit settings') ## ## Create some settings on the fly ## python_bin = sys.executable projectRoot = T.projectRoot() pymol_scripts = T.dataRoot() + '/pymol/' ################################### ## required environment variables. ## format: ENV_NAME : path_example ## Todo: These need to go to their Exe_*.dat files env = {} blast_env = { 'BLASTDB': '/home/Bis/raik/data/prog/blast/db', 'BLASTMA': '/home/Bis/johan/APPLICATIONS/blast' }