class LevelsTest(ut.TestCase):
def setUp(self):
self.dispatcher = Dispatcher('units.info', CONFIG, Session)
def levels(self, *args):
levels = []
for objs in self.dispatcher.levels(*args):
levels.append([obj.name for obj in objs])
return levels
def test_produces_correct_number_of_levels(self):
deps = self.dispatcher.dependencies([pdbs.Loader])
val = self.levels(deps, [], True)
assert val == [['pdbs.info'], ['pdbs.obsolete']]
def test_has_no_excluded_stages(self):
deps = self.dispatcher.dependencies([pdbs.Loader])
val = self.levels(deps, ['pdbs.obsolete'], [])
assert ['pdbs.obsolete'] not in val
assert val == [['pdbs.info']]
def test_will_always_have_allowed_stages(self):
deps = self.dispatcher.dependencies([pdbs.Loader])
val = self.levels(deps, ['pdbs.obsolete'], ['pdbs.obsolete'])
assert val == [['pdbs.info'], ['pdbs.obsolete']]
def explore(ctx, name, ids, recalculate=None, **kwargs):
kwargs.update(ctx.parent.objs)
engine = kwargs['engine']
config = kwargs['config']
mod.reflect(engine)
try:
setup.expand_stage_pattern(name, 'recalculate', kwargs)
setup.expand_stage_pattern(name, 'skip_stage', kwargs)
except introspect.UnknownStageError as err:
click.secho("Unknown stage %s" % err.args, err=True, fg='red')
ctx.exit(1)
if not ids:
ids = setup.pdbs(config, kwargs)
kwargs['exclude'] = kwargs.get('skip_stage')
dispatcher = Dispatcher(name, config, sessionmaker(engine), **kwargs)
result = []
for stage in dispatcher.stages(name):
recalc = ''
if recalculate is True or stage.name in recalculate:
recalc = 'Will Recalculate'
result.append((stage.name, recalc))
formatter = click.HelpFormatter(width=90)
formatter.write_dl(result)
click.echo(formatter.getvalue(), nl=False)
def explore(ctx, name, ids, recalculate=None, **kwargs):
kwargs.update(ctx.parent.objs)
engine = kwargs['engine']
config = kwargs['config']
mod.reflect(engine)
try:
setup.expand_stage_pattern(name, 'recalculate', kwargs)
setup.expand_stage_pattern(name, 'skip_stage', kwargs)
except introspect.UnknownStageError as err:
click.secho("Unknown stage %s" % err.args, err=True, fg='red')
ctx.exit(1)
if not ids:
ids = setup.pdbs(config, kwargs)
kwargs['exclude'] = kwargs.get('skip_stage')
dispatcher = Dispatcher(name, config, sessionmaker(engine), **kwargs)
result = []
for stage in dispatcher.stages(name):
recalc = ''
if recalculate is True or stage.name in recalculate:
recalc = 'Will Recalculate'
result.append((stage.name, recalc))
formatter = click.HelpFormatter(width=90)
formatter.write_dl(result)
click.echo(formatter.getvalue(), nl=False)
def test_can_exclude_stages_in_container(self):
dispatcher = Dispatcher('motifs.loader', CONFIG, Session,
skip_dependencies=True,
exclude=['motifs.release'])
val = [o.name for o in dispatcher.stages('motifs.loader')]
assert 'motifs.release' not in val
assert val == [
'motifs.discrepancies',
'motifs.info',
'motifs.assignments',
'motifs.loop_order',
'motifs.loop_positions',
]
class DependenciesTest(ut.TestCase):
def setUp(self):
self.dispatcher = Dispatcher('units.info', CONFIG, Session)
def test_it_can_compute_a_dependency_graph(self):
val = self.dispatcher.dependencies([units.InfoLoader])
assert val == {
units.InfoLoader: set([down.Downloader, pdbs.InfoLoader]),
pdbs.InfoLoader: set([]),
down.Downloader: set([]),
}
def test_it_computes_dependencies_for_stage_container(self):
val = self.dispatcher.dependencies([pdbs.Loader])
assert val == {
pdbs.InfoLoader: set([]),
pdbs.ObsoleteLoader: set([pdbs.InfoLoader]),
}
def test_it_computes_dependencies_for_several(self):
val = self.dispatcher.dependencies([pdbs.Loader, units.InfoLoader])
assert val == {
pdbs.InfoLoader: set([]),
down.Downloader: set([]),
pdbs.ObsoleteLoader: set([pdbs.InfoLoader]),
units.InfoLoader: set([down.Downloader, pdbs.InfoLoader]),
}
def test_it_computes_dependencies_for_nested_containers(self):
val = self.dispatcher.dependencies([ifes.Loader])
assert val[ifes.InfoLoader] == set([
units.InfoLoader,
units.DistancesLoader,
units.CenterLoader,
units.RotationLoader,
units.IncompleteLoader,
units.CoordinateLoader,
chains.InfoLoader,
chains.SpeciesLoader,
interactions.PairwiseLoader,
interactions.FlankingLoader,
interactions.SummaryLoader,
])
class ToExcludeTests(ut.TestCase):
def setUp(self):
self.dispatcher = Dispatcher('units.info', CONFIG, Session)
def test_given_nothing_returns_nothing(self):
assert self.dispatcher.to_exclude() == set()
def test_given_skip_dependencies_returns_true(self):
self.dispatcher.skip_dependencies = True
assert self.dispatcher.to_exclude(skip_dependencies=True) is True
val = self.dispatcher.to_exclude('units.info', skip_dependencies=True)
assert val is True
def test_will_dedup_excludes(self):
val = self.dispatcher.to_exclude('units.info', 'units.info')
assert val == set(['units.info'])
def test_will_complain_given_unknown_stage(self):
with pytest.raises(UnknownStageError):
self.dispatcher.to_exclude('bob')
def test_will_expand_stage_container_to_stages(self):
val = self.dispatcher.to_exclude('units.loader')
assert val == set(['units.info', 'units.distances',
'units.distances', 'units.centers', 'units.loader',
'units.rotation', 'units.incomplete',
'units.coordinates'])
@pytest.mark.xfail(reason='Cannot handle skip-deps and stage at once')
def test_can_exclude_stages_in_container(self):
dispatcher = Dispatcher('motifs.loader', CONFIG, Session,
skip_dependencies=True,
exclude=['motifs.release'])
val = [o.name for o in dispatcher.stages('motifs.loader')]
assert 'motifs.release' not in val
assert val == [
'motifs.discrepancies',
'motifs.info',
'motifs.assignments',
'motifs.loop_order',
'motifs.loop_positions',
]
def setUp(self):
self.dispatcher = Dispatcher('units.info', CONFIG, Session)
class StagesTest(ut.TestCase):
def setUp(self):
self.dispatcher = Dispatcher('units.info', CONFIG, Session)
def stages(self, *args):
return [o.name for o in self.dispatcher.stages(*args)]
def test_can_load_requested_stages(self):
assert self.stages('units.info') == [
'download',
'pdbs.info',
'units.info',
]
def test_when_skipping_produces_container_stages_in_correct_order(self):
self.dispatcher.skip_dependencies = True
assert self.stages('units.loader') == [
'units.info',
'units.centers',
'units.coordinates',
'units.distances',
'units.incomplete',
'units.rotation',
]
def test_it_can_load_stage_container(self):
assert self.stages('units.loader') == [
'download',
'pdbs.info',
'units.info',
'units.centers',
'units.coordinates',
'units.distances',
'units.incomplete',
'units.rotation',
]
def test_can_exclude_specific_stages(self):
self.dispatcher.exclude = set(['units.distances'])
val = self.stages('units.loader')
assert 'units.distances' not in val
assert val == [
'download',
'pdbs.info',
'units.info',
'units.centers',
'units.coordinates',
'units.incomplete',
'units.rotation',
]
def test_can_exclude_a_stage_collection(self):
self.dispatcher.exclude = set(['pdbs.loader', 'units.distances'])
assert self.stages('units.loader') == [
'download',
'units.info',
'units.centers',
'units.coordinates',
'units.incomplete',
'units.rotation',
]
@pytest.mark.xfail(reason="Haven't worked on yet")
def test_complains_if_nothing_to_run(self):
self.dispatcher.skip_dependencies = True
self.dispatcher.exclude.add('units.info')
with pytest.raises(core.InvalidState):
self.stages('units.info')
def test_can_get_one_stage_built(self):
self.dispatcher.skip_dependencies = True
assert self.stages('units.info') == ['units.info']
def test_running_with_nested_container_dependencies(self):
val = self.stages('ife.loader')
assert val.index('interactions.pairwise') < val.index('ife.info')
assert val == [
'download',
'pdbs.info',
'export.cifatom',
'pdbs.obsolete',
'units.info',
'chains.info',
'mat_files',
'units.centers',
'units.coordinates',
'units.distances',
'units.incomplete',
'units.rotation',
'interactions.flanking',
'interactions.pairwise',
'species_mapping',
'chains.species',
'interactions.summary',
'ife.info',
]
def run(ctx, name, ids, config=None, engine=None, **kwargs):
"""Actually run the pipeline. This is the main function which will load and
run the stages requested. It fetch PDBs to run if needed, reflect the
models, run stages, and send an email as needed.
Parameters
----------
name : str
The name of the stage to run.
ids : list
The PDB ids to use.
config : dict
The configuration dictionary as produced by pymotifs.config.load.
engine : Engine
The SQL Alchemy engine connection.
**kwargs : dict, optional
The other keyword arguments which will be passed to dispatcher.
"""
if kwargs.get('seed', None) is not None:
random.seed(kwargs['seed'])
mod.reflect(engine)
if kwargs.get('redo', False) is True:
kwargs['recalculate'] = '.'
kwargs['skip_dependencies'] = True
try:
setup.expand_stage_pattern(name, 'recalculate', kwargs)
setup.expand_stage_pattern(name, 'skip_stage', kwargs)
except introspect.UnknownStageError as err:
click.secho("Unknown stage %s" % err.args, err=True, fg='red')
ctx.exit(1)
# get desired PDB IDs
if not ids:
ids = setup.pdbs(config, kwargs)
logging.info("There are %d files listed to skip in skip_files.py" %
len(SKIP))
logging.info(
"There are %d files from skip_files.py that are also current PDB ids" %
len(SKIP & set(ids)))
logging.info(
"The following files in skip_files.py are not current PDB files")
logging.info(SKIP - set(ids))
# remove files from skip_files.py from the list of PDB IDs to process
ids = sorted(set(ids) - SKIP)
logging.info("Found %d files to process" % len(ids))
kwargs['exclude'] = kwargs.get('skip_stage')
logging.info("Running from command %s", ' '.join(sys.argv))
error = None
dispatcher = Dispatcher(name, config, sessionmaker(engine), **kwargs)
mailer = Emailer(config, engine)
try:
dispatcher(ids, **kwargs)
except Exception as error:
click.secho("Pipeline failed", fg='red', err=True)
logging.exception(error)
ctx.exit(1)
finally:
if kwargs['email']:
mailer(name, ids=ids, error=error, **kwargs)