class LevelsTest(ut.TestCase): def setUp(self): self.dispatcher = Dispatcher('units.info', CONFIG, Session) def levels(self, *args): levels = [] for objs in self.dispatcher.levels(*args): levels.append([obj.name for obj in objs]) return levels def test_produces_correct_number_of_levels(self): deps = self.dispatcher.dependencies([pdbs.Loader]) val = self.levels(deps, [], True) assert val == [['pdbs.info'], ['pdbs.obsolete']] def test_has_no_excluded_stages(self): deps = self.dispatcher.dependencies([pdbs.Loader]) val = self.levels(deps, ['pdbs.obsolete'], []) assert ['pdbs.obsolete'] not in val assert val == [['pdbs.info']] def test_will_always_have_allowed_stages(self): deps = self.dispatcher.dependencies([pdbs.Loader]) val = self.levels(deps, ['pdbs.obsolete'], ['pdbs.obsolete']) assert val == [['pdbs.info'], ['pdbs.obsolete']]
def explore(ctx, name, ids, recalculate=None, **kwargs): kwargs.update(ctx.parent.objs) engine = kwargs['engine'] config = kwargs['config'] mod.reflect(engine) try: setup.expand_stage_pattern(name, 'recalculate', kwargs) setup.expand_stage_pattern(name, 'skip_stage', kwargs) except introspect.UnknownStageError as err: click.secho("Unknown stage %s" % err.args, err=True, fg='red') ctx.exit(1) if not ids: ids = setup.pdbs(config, kwargs) kwargs['exclude'] = kwargs.get('skip_stage') dispatcher = Dispatcher(name, config, sessionmaker(engine), **kwargs) result = [] for stage in dispatcher.stages(name): recalc = '' if recalculate is True or stage.name in recalculate: recalc = 'Will Recalculate' result.append((stage.name, recalc)) formatter = click.HelpFormatter(width=90) formatter.write_dl(result) click.echo(formatter.getvalue(), nl=False)
def test_can_exclude_stages_in_container(self): dispatcher = Dispatcher('motifs.loader', CONFIG, Session, skip_dependencies=True, exclude=['motifs.release']) val = [o.name for o in dispatcher.stages('motifs.loader')] assert 'motifs.release' not in val assert val == [ 'motifs.discrepancies', 'motifs.info', 'motifs.assignments', 'motifs.loop_order', 'motifs.loop_positions', ]
class DependenciesTest(ut.TestCase): def setUp(self): self.dispatcher = Dispatcher('units.info', CONFIG, Session) def test_it_can_compute_a_dependency_graph(self): val = self.dispatcher.dependencies([units.InfoLoader]) assert val == { units.InfoLoader: set([down.Downloader, pdbs.InfoLoader]), pdbs.InfoLoader: set([]), down.Downloader: set([]), } def test_it_computes_dependencies_for_stage_container(self): val = self.dispatcher.dependencies([pdbs.Loader]) assert val == { pdbs.InfoLoader: set([]), pdbs.ObsoleteLoader: set([pdbs.InfoLoader]), } def test_it_computes_dependencies_for_several(self): val = self.dispatcher.dependencies([pdbs.Loader, units.InfoLoader]) assert val == { pdbs.InfoLoader: set([]), down.Downloader: set([]), pdbs.ObsoleteLoader: set([pdbs.InfoLoader]), units.InfoLoader: set([down.Downloader, pdbs.InfoLoader]), } def test_it_computes_dependencies_for_nested_containers(self): val = self.dispatcher.dependencies([ifes.Loader]) assert val[ifes.InfoLoader] == set([ units.InfoLoader, units.DistancesLoader, units.CenterLoader, units.RotationLoader, units.IncompleteLoader, units.CoordinateLoader, chains.InfoLoader, chains.SpeciesLoader, interactions.PairwiseLoader, interactions.FlankingLoader, interactions.SummaryLoader, ])
class ToExcludeTests(ut.TestCase): def setUp(self): self.dispatcher = Dispatcher('units.info', CONFIG, Session) def test_given_nothing_returns_nothing(self): assert self.dispatcher.to_exclude() == set() def test_given_skip_dependencies_returns_true(self): self.dispatcher.skip_dependencies = True assert self.dispatcher.to_exclude(skip_dependencies=True) is True val = self.dispatcher.to_exclude('units.info', skip_dependencies=True) assert val is True def test_will_dedup_excludes(self): val = self.dispatcher.to_exclude('units.info', 'units.info') assert val == set(['units.info']) def test_will_complain_given_unknown_stage(self): with pytest.raises(UnknownStageError): self.dispatcher.to_exclude('bob') def test_will_expand_stage_container_to_stages(self): val = self.dispatcher.to_exclude('units.loader') assert val == set(['units.info', 'units.distances', 'units.distances', 'units.centers', 'units.loader', 'units.rotation', 'units.incomplete', 'units.coordinates']) @pytest.mark.xfail(reason='Cannot handle skip-deps and stage at once') def test_can_exclude_stages_in_container(self): dispatcher = Dispatcher('motifs.loader', CONFIG, Session, skip_dependencies=True, exclude=['motifs.release']) val = [o.name for o in dispatcher.stages('motifs.loader')] assert 'motifs.release' not in val assert val == [ 'motifs.discrepancies', 'motifs.info', 'motifs.assignments', 'motifs.loop_order', 'motifs.loop_positions', ]
def setUp(self): self.dispatcher = Dispatcher('units.info', CONFIG, Session)
class StagesTest(ut.TestCase): def setUp(self): self.dispatcher = Dispatcher('units.info', CONFIG, Session) def stages(self, *args): return [o.name for o in self.dispatcher.stages(*args)] def test_can_load_requested_stages(self): assert self.stages('units.info') == [ 'download', 'pdbs.info', 'units.info', ] def test_when_skipping_produces_container_stages_in_correct_order(self): self.dispatcher.skip_dependencies = True assert self.stages('units.loader') == [ 'units.info', 'units.centers', 'units.coordinates', 'units.distances', 'units.incomplete', 'units.rotation', ] def test_it_can_load_stage_container(self): assert self.stages('units.loader') == [ 'download', 'pdbs.info', 'units.info', 'units.centers', 'units.coordinates', 'units.distances', 'units.incomplete', 'units.rotation', ] def test_can_exclude_specific_stages(self): self.dispatcher.exclude = set(['units.distances']) val = self.stages('units.loader') assert 'units.distances' not in val assert val == [ 'download', 'pdbs.info', 'units.info', 'units.centers', 'units.coordinates', 'units.incomplete', 'units.rotation', ] def test_can_exclude_a_stage_collection(self): self.dispatcher.exclude = set(['pdbs.loader', 'units.distances']) assert self.stages('units.loader') == [ 'download', 'units.info', 'units.centers', 'units.coordinates', 'units.incomplete', 'units.rotation', ] @pytest.mark.xfail(reason="Haven't worked on yet") def test_complains_if_nothing_to_run(self): self.dispatcher.skip_dependencies = True self.dispatcher.exclude.add('units.info') with pytest.raises(core.InvalidState): self.stages('units.info') def test_can_get_one_stage_built(self): self.dispatcher.skip_dependencies = True assert self.stages('units.info') == ['units.info'] def test_running_with_nested_container_dependencies(self): val = self.stages('ife.loader') assert val.index('interactions.pairwise') < val.index('ife.info') assert val == [ 'download', 'pdbs.info', 'export.cifatom', 'pdbs.obsolete', 'units.info', 'chains.info', 'mat_files', 'units.centers', 'units.coordinates', 'units.distances', 'units.incomplete', 'units.rotation', 'interactions.flanking', 'interactions.pairwise', 'species_mapping', 'chains.species', 'interactions.summary', 'ife.info', ]
def run(ctx, name, ids, config=None, engine=None, **kwargs): """Actually run the pipeline. This is the main function which will load and run the stages requested. It fetch PDBs to run if needed, reflect the models, run stages, and send an email as needed. Parameters ---------- name : str The name of the stage to run. ids : list The PDB ids to use. config : dict The configuration dictionary as produced by pymotifs.config.load. engine : Engine The SQL Alchemy engine connection. **kwargs : dict, optional The other keyword arguments which will be passed to dispatcher. """ if kwargs.get('seed', None) is not None: random.seed(kwargs['seed']) mod.reflect(engine) if kwargs.get('redo', False) is True: kwargs['recalculate'] = '.' kwargs['skip_dependencies'] = True try: setup.expand_stage_pattern(name, 'recalculate', kwargs) setup.expand_stage_pattern(name, 'skip_stage', kwargs) except introspect.UnknownStageError as err: click.secho("Unknown stage %s" % err.args, err=True, fg='red') ctx.exit(1) # get desired PDB IDs if not ids: ids = setup.pdbs(config, kwargs) logging.info("There are %d files listed to skip in skip_files.py" % len(SKIP)) logging.info( "There are %d files from skip_files.py that are also current PDB ids" % len(SKIP & set(ids))) logging.info( "The following files in skip_files.py are not current PDB files") logging.info(SKIP - set(ids)) # remove files from skip_files.py from the list of PDB IDs to process ids = sorted(set(ids) - SKIP) logging.info("Found %d files to process" % len(ids)) kwargs['exclude'] = kwargs.get('skip_stage') logging.info("Running from command %s", ' '.join(sys.argv)) error = None dispatcher = Dispatcher(name, config, sessionmaker(engine), **kwargs) mailer = Emailer(config, engine) try: dispatcher(ids, **kwargs) except Exception as error: click.secho("Pipeline failed", fg='red', err=True) logging.exception(error) ctx.exit(1) finally: if kwargs['email']: mailer(name, ids=ids, error=error, **kwargs)