class builder(Node):
"""Builds and adds a :term:`SKETCH` from given :term:`FORM` string to the :class:`.Case` using ideal SSE elements.
If corrections are available and specified, these will be applied onto the :term:`SKETCH`.
.. caution::
In order to apply secondary structure or per layer corrections, the :mod:`.corrector` plugin
needs to be set in the :class:`.Pipeline`.
:param connectivity: Expected secondary structure connectivity. *Important*: at the moment only a single
connectivity supported (default: True).
:param motif: Expected Motif to be added to the :term:`SKETCH` (default: False).
:param pick_aa: Desired amino acid type to use for the :term:`SKETCH` sequence. If not specified, it will
use pseudorandomly assign amino acid types based on secondary structure propensity scores.
:param write2disc: Dump the :term:`SKETCH` (default: True).
:raises:
:NodeDataError: On **check**. If the required fields to be executed are not there.
:NodeDataError: On **execution**. If the :class:`.Case` contains anything other than one defined connectivity.
"""
REQUIRED_FIELDS = ('topology.architecture', 'topology.connectivity')
RETURNED_FIELDS = ()
VERSION = 'v1.0'
def __init__(self,
tag: int,
connectivity: Optional[bool] = True,
motif: Optional[bool] = False,
pick_aa: Optional[str] = None,
write2disc: Optional[str] = True):
super(builder, self).__init__(tag)
self.connectivity = connectivity
self.motif = motif
self.pick_aa = pick_aa
self.write2disc = write2disc
def single_check(self, dummy: Dict) -> Dict:
kase = Case(dummy)
# Check what it needs
for itag in self.REQUIRED_FIELDS:
if kase[itag] is None:
raise NodeDataError(f'Field "{itag}" is required')
# Include what keywords it adds (in this instance, nothing)
# Here Nothing
return kase.data
def single_execute(self, data: Dict) -> Dict:
case = Case(data)
# Apply connectivity?
if self.connectivity:
if case.connectivity_count == 0:
raise NodeDataError(
'Minimum a single connectivity must be provided.')
if case.connectivity_count > 1:
raise NodeDataError(
'Only single connectivity cases can be build.')
def pds_database( log: Logger,
filter: Optional[Union[str, Path]] = None,
) -> Tuple[Path, List]:
"""Provide the list of target PDS as a file and a list.
.. note::
Depends on the ``master.pds`` configuration option.
:param log: Job logger.
:param filter: File containting the target subset, otherwise all PDS database is considered.
"""
# @TODO: PDS-FILTER
pds_file = Path(TBcore.get_option('master', 'pds'))
if pds_file.is_file():
pds_list = [line.strip() for line in open(pds_file).readlines() if len(line.strip()) > 0]
elif pds_file.is_dir():
pds_list = [str(x.resolve()) for x in pds_file.glob('*/*.pds')]
else:
raise NodeDataError('The provided MASTER database directory/list file cannot be found.')
# Even if a PDS file already exists, we always create a temporary file so that we can
# manage different versions of the PDS database in different Nodes.
f = NamedTemporaryFile(mode='w', delete=False)
log.info(f'Temporary file for PDS database: {f.name}')
[f.write(x + '\n') for x in pds_list]
f.close()
pds_file = Path(f.name)
return pds_file, pds_list
def single_check(self, dummy: Dict) -> Dict:
kase = Case(dummy)
# Check what it needs
for itag in self.REQUIRED_FIELDS:
if kase[itag] is None:
raise NodeDataError(f'Field "{itag}" is required')
# Include what keywords it adds
kase.data.setdefault('metadata', {}).setdefault('motif_picker', [])
return kase.data
def single_check(self, dummy: Dict) -> Dict:
kase = Case(dummy)
# Check what it needs
for itag in self.REQUIRED_FIELDS:
if kase[itag] is None:
raise NodeDataError(f'Field "{itag}" is required')
# Include what keywords it adds (in this instance, nothing)
return kase.data
def get_fragfiles(self) -> pd.DataFrame:
"""Obtain the fragment files.
"""
fragpath = Path(core.get_option('loop_master', 'fragments'))
self.log.debug(f'Listing available fragment files at: {fragpath.name}')
if not fragpath.is_dir():
raise NodeDataError(f'{fragpath.name} is not a folder.')
return pd.DataFrame(
[(x.name[:4], x.name[5:6], x, y)
for x, y in zip(sorted(fragpath.glob('*/*3mers.gz')),
sorted(fragpath.glob('*/*9mers.gz')))],
columns=['pdb', 'chain', '3mers', '9mers'])
def single_check(self, dummy: Dict) -> Dict:
kase = Case(dummy)
# Check what it needs
for itag in self.REQUIRED_FIELDS:
if kase[itag] is None:
raise NodeDataError(f'Field "{itag}" is required')
# Include what keywords it adds (in this instance, nothing)
kase.data.setdefault('metadata', {}).setdefault('loop_fragments', [])
kase.data.setdefault('metadata', {}).setdefault('loop_lengths', [])
return kase.data
class fragment_maker(Node):
"""Creates or mixes fragments that are needed in multiple Rosetta protocols. Mutliple ways of creating fragments
are possible through different protocols.
.. note::
Currently, solely the ``loop_fragment`` protocol is implemented.
.. caution::
In order to create fragments with the ``loop_fragment`` protocol, the :mod:`.loop_fragments` plugin
needs to be set in the :class:`.Pipeline`.
:param protocol: Fragment creation protocol to be used.
:param script: Rosetta script to pick fragments.
:raises:
:NodeDataError: On **check**. If the required fields to be executed are not there.
:NodeDataError: On **execution**. If the :class:`.Case` contains anything other than one defined connectivity.
:NodeMissingError: On **exection**. If required variable inputs are not there.
"""
REQUIRED_FIELDS = ('metadata.loop_fragments', 'metadata.loop_lengths')
RETURNED_FIELDS = ('metadata.fragments')
VERSION = 'v1.0'
def __init__(self,
tag: int,
protocol: str,
script: Optional[Union[Path, str]] = None):
super(fragment_maker, self).__init__(tag)
self.protocol = protocol
self.script = script
def single_check(self, dummy: Dict) -> Dict:
kase = Case(dummy)
# Check what it needs
for itag in self.REQUIRED_FIELDS:
if kase[itag] is None:
raise NodeDataError(f'Field "{itag}" is required')
# Include what keywords it adds (in this instance, nothing)
kase.data.setdefault('metadata', {}).setdefault('fragments', {})
return kase.data
def single_execute(self, data: Dict) -> Dict:
case = Case(data)
data = {'protocol': self.protocol, 'files': []}
# Fragments can only be made for a full, reoriented Case.
if case.connectivity_count > 1:
raise NodeDataError(
'FunFolDes can only be applied to one connectivity.')
def createPDS( infile: Union[Path, str], outfile: Optional[str] = None ) -> List[str]:
"""Make the createPDS command call.
.. note::
Depends on the ``master.create`` configuration option.
:param infile: PDB file to convert.
:param outfile: Name of the expected PDS output.
"""
_, createPDS = get_master_exes()
infile = Path(infile)
if not infile.is_file():
raise NodeDataError(f'Unable to find structure file {infile}')
outfile = outfile if outfile is not None else infile.with_suffix('.pds')
return shlex.split(f'{createPDS} --type query --pdb {str(infile)} --pds {str(outfile)} --dCut 100')
def get_abegos(self) -> pd.DataFrame:
"""Load ABEGO data.
"""
abegos = Path(core.get_option('loop_master', 'abego'))
if not abegos.is_file():
raise NodeDataError(f'ABEGO file {abegos.name} cannot be found.')
self.log.debug(f'Loading ABEGO data from: {abegos.name}\n')
doopen = gzip.open if abegos.suffix == '.gz' else open
abegodata = []
with doopen(abegos, 'rt') as fd:
for line1, line2 in itertools.zip_longest(*[fd] * 2):
line2 = line2 if len(line2.strip()) != 0 else 'NON\n'
line1 = line1.strip().lstrip('>').split('_')
abegodata.append(f'{line1[0]},{line1[1]},{line2}')
abegodata = pd.read_csv(StringIO(''.join(abegodata)),
names=['pdb', 'chain', 'abego'],
header=None)
abegodata = abegodata[abegodata['abego'] != 'NON']
return abegodata
def single_execute(self, data: Dict) -> Dict:
kase = Case(data)
result = {'id': f'motif_{self.identifier}'}
# Create a working folder
folders = kase.undirected_path.joinpath(result['id'])
folders.mkdir(parents=True, exist_ok=True)
result['data_dir'] = str(folders)
# Load Structure and create eigens
try:
motifs = self.Motif(*reverse_motif(
self.log, self.source, self.selection, self.attach,
self.hotspot, self.identifier, self.binder))
except StructuralError as se:
raise NodeDataError(str(se))
result['motifs'] = motifs
# Attach data and return
kase.data.setdefault('metadata', {}).setdefault('motif_picker',
[]).append(result)
return kase.data
class statistics( Node ):
"""Various statistics on the sequence and structure level are computed depending on available scripts.
.. note::
Depends on the ``statistic.molprobity`` configuration option.
Depends on the ``statistic.tmalign`` configuration option.
Depends on the ``statistic.trrosetta_repo`` configuration option.
Depends on the ``statistic.trrosetta_wts`` configuration option.
Depends on the ``statistic.trrosetta_env`` configuration option.
.. caution::
In order to execute this :class:`.Node`, we highly recommend to install `trRosetta` with all dependencies.
The external conda environment can be specified in the ``statistic.trrosetta_env`` configuration option.
.. admonition:: To Developers
Due to its use in multiple :class:`.Node`, functions to deal with this :class:`.Node` are mostly located
in the respective module file and external scripts are locate in this :class:`.Node` directory.
:param loop_range: Expected loop length is calculated from the euclidian distance between two secondary
structures. This attribute adds a window of ``loop_range`` residues under and over the calculated
length.
:param source: Plugin designs come from, e.g. :class:`funfoldes`.
:param stage: The type of design, e.g. folded or designed.
:param analysis: Geometric or quality assessment.
:param metric: Type of geometric or quality assessment.
:raises:
:NodeDataError: On **check**. If the required fields to be executed are not there.
"""
REQUIRED_FIELDS = ()
RETURNED_FIELDS = ()
VERSION = 'v1.0'
def __init__( self, tag: int,
source: str,
stage: str,
analysis: str,
metric: Optional[str] = None,
**kwargs ) -> str:
super(statistics, self).__init__(tag)
self.source = source
self.stage = stage
self.analysis = analysis
self.metric = metric
def single_check( self, dummy: Dict ) -> Dict:
case = Case(dummy)
# Check what it needs
for itag in self.REQUIRED_FIELDS:
if case[itag] is None:
raise NodeDataError(f'Field "{itag}" is required')
# Include what keywords it adds (in this instance, nothing)
if self.analysis == 'geometry':
case['metadata'].setdefault('statistic', {}).setdefault('geometry', '')
if self.analysis == 'quality':
case['metadata'].setdefault('statistic', {}).setdefault('quality', '')
return case.data
def funfoldes2pdb( self, case: Case, wfolder: Path ) -> List:
"""
"""
if self.stage == 'folding':
silent_files = case['metadata.funfoldes.silent_files.folding']
elif self.stage == 'design':
silent_files = case['metadata.funfoldes.silent_files.design']
if silent_files is None:
raise NodeMissingError('There is no output data from the funfoldes plugin.')
extract_pdb = Path(str(Path(TBcore.get_option('rosetta', 'scripts')).resolve()
).replace('rosetta_scripts.', 'extract_pdbs.'))
if not extract_pdb.is_file() or not os.access(str(extract_pdb), os.X_OK):
raise NodeDataError(f'Cannot find executable {extract_pdb}')
if not TBcore.get_option('slurm', 'use'):
cmd = [extract_pdb, '-in:file:silent']
cmd.extend(silent_files)
cmd.extend(['-out:prefix', str(wfolder) + '/'])
else:
indir = str(wfolder.joinpath('${SLURM_ARRAY_TASK_ID}'))
cmd = ['srun', extract_pdb, '-in:file:silent']
if self.stage == 'folding':
cmd.append(os.path.commonprefix([str(x) for x in silent_files]) + '${SLURM_ARRAY_TASK_ID}_funfol.silent')
elif self.stage == 'design':
cmd.append(os.path.commonprefix([str(x) for x in silent_files]) + '${SLURM_ARRAY_TASK_ID}_des.silent')
cmd.extend(['-out:prefix', str(indir) + '/'])
return [['mkdir', '-p', indir] if TBcore.get_option('slurm', 'use') else '', cmd]
def single_execute(self, data: Dict) -> Dict:
kase = Case(data)
# Loop MASTER is only applied to a Case with one single connectivity
if kase.connectivity_count != 1:
err = f'{self.nodeID} can only be applied to one connectivity. '
err += f'Current case contains a total of {kase.connectivity_count}.'
raise NodeDataError(err)
# And has to be reoriented
if not kase.is_reoriented:
self.log.debug(
'Topology was provided without oriented SSE -> orienting.')
kase = kase.apply_topologies()[0]
# Generate the folder tree for a single connectivity.
folders = kase.connectivities_paths[0].joinpath('loopgroup_master')
folders.mkdir(parents=True, exist_ok=True)
# Global step distance
loop_step = kase.cast_absolute(
)['configuration.defaults.distance.loop_step']
# Output keys
kase.data.setdefault('metadata', {}).setdefault('loop_fragments', [])
kase.data.setdefault('metadata', {}).setdefault('loop_lengths', [])
# Find steps: Each pair of secondary structure.
#it = kase.connectivities_str[0].split('.')
#steps = [it[i:i + 2] for i in range(0, len(it) - 1)]
lengths = kase.connectivity_len[0]
start = 1
for i, (group, infos) in enumerate(self.steps.items()):
self.log.info(f'Search at: {group}')
# 1. Make folders and files
wfolder = folders.joinpath(f'loopgroup{i + 1:02d}')
wfolder.mkdir(parents=True, exist_ok=True)
outfile = wfolder.joinpath(f'loopgroup_master.iter{i + 1:02d}.pdb')
outfilePDS = wfolder.joinpath(
f'loopgroup_master.iter{i + 1:02d}.pds')
masfile = outfile.with_suffix('.master')
gr = self.steps[f'group{i + 1:02d}'][-1].split(';')
gr = [int(g) for g in gr if g != 'x']
for g in gr:
checkpoint = wfolder.joinpath(f'loop{g:02d}/checkpoint.json')
# 2. Check if checkpoint exists, retrieve and skip
reload = TButil.checkpoint_in(self.log, checkpoint)
if reload is not None:
self.log.debug(
f'Reloading loopgroup{i + 1:02d} with loop{g:02d}')
kase.data['metadata']['loop_fragments'].append(reload)
kase.data['metadata']['loop_lengths'].append(
int(reload['edges']['loop']))
start += (int(reload['edges']['sse1']) +
int(reload['edges']['loop']))
continue
# 3. Check hairpin
# Get SSEs and identifiers
sses = [kase.get_sse_by_id(sse) for sse in infos[0]]
#sse1_name, sse2_name = sse1['id'], sse2['id']
#is_hairpin = self.check_hairpin(sse1_name, sse2_name)
# 4. Generate structures
sses = TBstructure.build_pdb_object(self.log,
sses,
5,
concat=False,
outfile=outfile)
if not masfile.is_file():
# 5. calculate expected loop length by loop_step
#Mdis, mdis = TBstructure.get_loop_length(self.log, sse1, sse2, loop_step, self.loop_range)
# 6. Run MASTER
#outfilePDS = outfile if outfile is not None else Path(outfile).with_suffix('.pds')
self.log.debug(f'FILE {outfilePDS}')
# -> make PDS query
cmd = TBMaster.createPDS(outfile, outfilePDS)
self.log.debug(f'EXECUTE: {" ".join(cmd)}')
run(cmd, stdout=DEVNULL)
# -> run MASTER
cmd = TBMaster.master_groupedgap(outfilePDS, self.pdsdb,
masfile, infos[1],
self.rmsd_cut)
self.log.debug(f'EXECUTE: {" ".join(cmd)}')
result = run(cmd, stdout=DEVNULL)
# TODO: implement motif compability
# if result.returncode: # no loop between that connection, e.g. a motif ranging over multiple sse with keeping the loops
# # 4. Generate structures
# self.log.debug('generate combined structure')
# sse = pd.concat([sse1, sse2], sort=False)
#
# # 6. Run MASTER
# self.log.debug(Path(outfile))
# #outfilePDS = outfile if outfile is not None else Path(outfile).with_suffix('.pds')
# self.log.debug(f'FILE {outfilePDS}')
# # -> make PDS query
# cmd = TBMaster.createPDS(outfile, outfilePDS)
# self.log.debug(f'EXECUTE: {" ".join(cmd)}')
# run(cmd, stdout=DEVNULL)
# # -> run MASTER
# cmd = TBMaster.master_nogap(outfilePDS, self.pdsdb, masfile, self.rmsd_cut)
# self.log.debug(f'EXECUTE: {" ".join(cmd)}')
# run(cmd, stdout=DEVNULL)
#
# # 6. Minimize master data (pick top_loopsx3 lines to read and minimize the files)
# match_count = self.minimize_master_file(masfile)
# self.log.debug(f'match count here {match_count}')
#
# # 7. Retrieve MASTER data
# dfloop = self.process_master_data_no_gap(masfile, sse1_name, sse2_name)
# sse1l, loopl, sse2l = lengths[i], int(dfloop['loop_length'].values[0]), lengths[i + 1]
# total_len = sse1l + loopl + sse2l
# end_edge = total_len + start - 1
# edges = {'ini': int(start), 'end': int(end_edge), 'sse1': int(sse1l), 'loop': int(loopl), 'sse2': int(sse2l)}
# self.log.debug(f'INI: {start}; END: {end_edge}; SSE1: {sse1l}; LOOP: {loopl}; SSE2: {sse2l}')
# self.log.debug(dfloop.to_string())
#
# # 8. Bring and Combine fragments from the different sources.
# loop_data = self.make_fragment_files(dfloop, edges, masfile)
# loop_data['match_count'] += match_count
#else:
# 6. Minimize master data (pick top_loopsx3 lines to read and minimize the files)
match_count = self.minimize_master_file(masfile)
# 7. Retrieve MASTER data
df_container = self.process_master_data(
masfile, infos[0], infos[1], infos[2])
for indx in list(df_container.order.drop_duplicates()):
dfloop = df_container[df_container.order == indx]
sse1l, loopl, sse2l = lengths[i], int(
dfloop['loop_length'].values[0]), lengths[i + 1]
total_len = sse1l + loopl + sse2l
end_edge = total_len + start - 1
edges = {
'ini': int(start),
'end': int(end_edge),
'sse1': int(sse1l),
'loop': int(loopl),
'sse2': int(sse2l)
}
self.log.debug(
f'INI: {start}; END: {end_edge}; SSE1: {sse1l}; LOOP: {loopl}; SSE2: {sse2l}'
)
self.log.debug(dfloop.to_string())
# 8. Bring and Combine fragments from the different sources.
loop_data, nfolder = self.make_fragment_files(dfloop,
edges,
masfile,
no_loop=True)
loop_data['match_count'] += match_count
# 9. Save data in the Case
kase.data['metadata']['loop_fragments'].append(loop_data)
kase.data['metadata']['loop_lengths'].append(int(loopl))
start += (sse1l + loopl)
# 10. Checkpoint save
checkpoint = nfolder.joinpath('checkpoint.json')
TButil.checkpoint_out(self.log, checkpoint, loop_data)
return kase
ofile = case.main_path.joinpath('architecture').joinpath(
'undirected_sketch.pdb')
# for i, j, sse in case:
# if 'atoms' in sse['metadata'] and not TBcore.get_option('system', 'overwrite'):
# self.log.debug(f'{case.name}.{sse["id"]} already has atoms defined\n')
# continue
#
# self.log.debug(f'Building coordinates for {case.name}.{sse["id"]}\n')
# case.data['topology']['architecture'][i][j] = self.make_structure(sse, self.pick_aa)
# Insert motif?
if self.motif:
if not case.data['metadata']['motif_picker']:
raise NodeDataError(
'Motif must be provided and pre-picked through the motif_picker.'
)
else:
# Include what keywords it adds (in this instance, nothing)
case.data.setdefault('metadata', {}).setdefault('binder', {})
for n, motif_data in enumerate(
case.data['metadata']['motif_picker']):
motif, binder, hotspots, attach, selection, identifier = motif_data[
'motifs']
motif = pd.DataFrame(motif,
columns=[
'auth_comp_id', 'auth_atom_id',
'auth_seq_id', 'auth_asym_id',
'sse_id', 'internal_num',
'Cartn_x', 'Cartn_y', 'Cartn_z'