def explore(ctx, name, ids, recalculate=None, **kwargs):
kwargs.update(ctx.parent.objs)
engine = kwargs['engine']
config = kwargs['config']
mod.reflect(engine)
try:
setup.expand_stage_pattern(name, 'recalculate', kwargs)
setup.expand_stage_pattern(name, 'skip_stage', kwargs)
except introspect.UnknownStageError as err:
click.secho("Unknown stage %s" % err.args, err=True, fg='red')
ctx.exit(1)
if not ids:
ids = setup.pdbs(config, kwargs)
kwargs['exclude'] = kwargs.get('skip_stage')
dispatcher = Dispatcher(name, config, sessionmaker(engine), **kwargs)
result = []
for stage in dispatcher.stages(name):
recalc = ''
if recalculate is True or stage.name in recalculate:
recalc = 'Will Recalculate'
result.append((stage.name, recalc))
formatter = click.HelpFormatter(width=90)
formatter.write_dl(result)
click.echo(formatter.getvalue(), nl=False)
def explore(ctx, name, ids, recalculate=None, **kwargs):
kwargs.update(ctx.parent.objs)
engine = kwargs['engine']
config = kwargs['config']
mod.reflect(engine)
try:
setup.expand_stage_pattern(name, 'recalculate', kwargs)
setup.expand_stage_pattern(name, 'skip_stage', kwargs)
except introspect.UnknownStageError as err:
click.secho("Unknown stage %s" % err.args, err=True, fg='red')
ctx.exit(1)
if not ids:
ids = setup.pdbs(config, kwargs)
kwargs['exclude'] = kwargs.get('skip_stage')
dispatcher = Dispatcher(name, config, sessionmaker(engine), **kwargs)
result = []
for stage in dispatcher.stages(name):
recalc = ''
if recalculate is True or stage.name in recalculate:
recalc = 'Will Recalculate'
result.append((stage.name, recalc))
formatter = click.HelpFormatter(width=90)
formatter.write_dl(result)
click.echo(formatter.getvalue(), nl=False)
def units(column, **kwargs):
mod.reflect(kwargs['engine'])
table_name, name = column.split('.')
table_name = mod.camelize_classname(table_name)
table = getattr(mod, table_name)
session = Session(sessionmaker(kwargs['engine']))
corrector = TableCorrector(kwargs['config'], session)
corrector(table, name, **kwargs)
def list(ctx, **kwargs):
"""This will output each stage name along with the short documentation on
the stage, if any is available. This will only list stages which are part
of the 'update' stage.
"""
mod.reflect(ctx.parent.objs['engine'])
formatter = click.HelpFormatter(width=90)
formatter.write_dl((s[0], s[1]) for s in introspect.stages())
click.echo(formatter.getvalue(), nl=False)
def list(ctx, **kwargs):
"""This will output each stage name along with the short documentation on
the stage, if any is available. This will only list stages which are part
of the 'update' stage.
"""
mod.reflect(ctx.parent.objs['engine'])
formatter = click.HelpFormatter(width=90)
formatter.write_dl((s[0], s[1]) for s in introspect.stages())
click.echo(formatter.getvalue(), nl=False)
def run(ctx, name, ids, config=None, engine=None, **kwargs):
"""Actually run the pipeline. This is the main function which will load and
run the stages requested. It fetch PDBs to run if needed, reflect the
models, run stages, and send an email as needed.
Parameters
----------
name : str
The name of the stage to run.
ids : list
The PDB ids to use.
config : dict
The configuration dictionary as produced by pymotifs.config.load.
engine : Engine
The SQL Alchemy engine connection.
**kwargs : dict, optional
The other keyword arguments which will be passed to dispatcher.
"""
if kwargs.get('seed', None) is not None:
random.seed(kwargs['seed'])
mod.reflect(engine)
if kwargs.get('redo', False) is True:
kwargs['recalculate'] = '.'
kwargs['skip_dependencies'] = True
try:
setup.expand_stage_pattern(name, 'recalculate', kwargs)
setup.expand_stage_pattern(name, 'skip_stage', kwargs)
except introspect.UnknownStageError as err:
click.secho("Unknown stage %s" % err.args, err=True, fg='red')
ctx.exit(1)
if not ids:
ids = setup.pdbs(config, kwargs)
kwargs['exclude'] = kwargs.get('skip_stage')
logging.info("Running from commadn %s", ' '.join(sys.argv))
error = None
dispatcher = Dispatcher(name, config, sessionmaker(engine), **kwargs)
mailer = Emailer(config, engine)
try:
dispatcher(ids, **kwargs)
except Exception as error:
click.secho("Pipeline failed", fg='red', err=True)
logging.exception(error)
ctx.exit(1)
finally:
if kwargs['email']:
mailer(name, ids=ids, error=error, **kwargs)
def setup(engine, **kwargs):
"""Reflect the models and create a session wrapper.
Parameters
----------
engine : Engine
An engine to use to reflect models.
Returns
session : pymotifs.core.Session
The session wrapper to use.
"""
mod.reflect(engine)
return Session(sessionmaker(engine))
def setup(engine, **kwargs):
"""Reflect the models and create a session wrapper.
Parameters
----------
engine : Engine
An engine to use to reflect models.
Returns
session : pymotifs.core.Session
The session wrapper to use.
"""
mod.reflect(engine)
return Session(sessionmaker(engine))
def about(ctx, name=None, **kwargs):
"""Display help for a stage. This will write out the complete documentation
for each stage with the given name.
"""
mod.reflect(ctx.parent.objs['engine'])
info = introspect.get_stage_info(name)
if not info:
click.echo("Unknown stage %s" % name, err=True, fg='red')
ctx.exit(1)
formatter = click.HelpFormatter()
with formatter.section(name=name):
formatter.write_text(info[2])
click.echo(formatter.getvalue())
def about(ctx, name=None, **kwargs):
"""Display help for a stage. This will write out the complete documentation
for each stage with the given name.
"""
mod.reflect(ctx.parent.objs['engine'])
info = introspect.get_stage_info(name)
if not info:
click.echo("Unknown stage %s" % name, err=True, fg='red')
ctx.exit(1)
formatter = click.HelpFormatter()
with formatter.section(name=name):
formatter.write_text(info[2])
click.echo(formatter.getvalue())
def run(ctx, name, ids, config=None, engine=None, **kwargs):
"""Actually run the pipeline. This is the main function which will load and
run the stages requested. It fetch PDBs to run if needed, reflect the
models, run stages, and send an email as needed.
Parameters
----------
name : str
The name of the stage to run.
ids : list
The PDB ids to use.
config : dict
The configuration dictionary as produced by pymotifs.config.load.
engine : Engine
The SQL Alchemy engine connection.
**kwargs : dict, optional
The other keyword arguments which will be passed to dispatcher.
"""
if kwargs.get('seed', None) is not None:
random.seed(kwargs['seed'])
mod.reflect(engine)
if kwargs.get('redo', False) is True:
kwargs['recalculate'] = '.'
kwargs['skip_dependencies'] = True
try:
setup.expand_stage_pattern(name, 'recalculate', kwargs)
setup.expand_stage_pattern(name, 'skip_stage', kwargs)
except introspect.UnknownStageError as err:
click.secho("Unknown stage %s" % err.args, err=True, fg='red')
ctx.exit(1)
# get desired PDB IDs
if not ids:
ids = setup.pdbs(config, kwargs)
logging.info("There are %d files listed to skip in skip_files.py" %
len(SKIP))
logging.info(
"There are %d files from skip_files.py that are also current PDB ids" %
len(SKIP & set(ids)))
logging.info(
"The following files in skip_files.py are not current PDB files")
logging.info(SKIP - set(ids))
# remove files from skip_files.py from the list of PDB IDs to process
ids = sorted(set(ids) - SKIP)
logging.info("Found %d files to process" % len(ids))
kwargs['exclude'] = kwargs.get('skip_stage')
logging.info("Running from command %s", ' '.join(sys.argv))
error = None
dispatcher = Dispatcher(name, config, sessionmaker(engine), **kwargs)
mailer = Emailer(config, engine)
try:
dispatcher(ids, **kwargs)
except Exception as error:
click.secho("Pipeline failed", fg='red', err=True)
logging.exception(error)
ctx.exit(1)
finally:
if kwargs['email']:
mailer(name, ids=ids, error=error, **kwargs)
import unittest as ut
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from fr3d.cif.reader import Cif
import pymotifs.models as models
from pymotifs.config import load as config_loader
from pymotifs.utils.matlab import exists as has_matlab
CONFIG = config_loader('conf/bootstrap.json')
engine = create_engine(CONFIG['db']['uri'])
models.reflect(engine)
Session = sessionmaker(bind=engine)
skip_without_matlab = pytest.mark.skipif(has_matlab() is False,
reason="No matlab installed")
class StageTest(ut.TestCase):
loader_class = None
def setUp(self):
os.chdir(CONFIG['locations']['base'])
if self.loader_class:
self.loader = self.loader_class(CONFIG, Session)
def nr_history(version, **kwargs):
mod.reflect(kwargs['engine'])
session = Session(sessionmaker(kwargs['engine']))