def task_host():
""" Start a web server hosting the contents of the output directory.
"""
cmd = "cd {} && python -m http.server".format(output_dir)
return {
"actions" : [LongRunning(cmd)]
}
def get_download_task(url, target_fn, md5=None, metalink=None):
'''Creates a doit task to download the given URL.
Args:
url (str): URL to download.
target_fn (str): Target for the download.
Returns:
dict: doit task.
'''
cmd = ['curl', '-o', target_fn]
if metalink is not None:
cmd.extend(['--metalink', metalink])
cmd.append(url)
cmd = ' '.join(cmd)
name = 'download:{0}'.format(os.path.basename(target_fn))
actions = [LongRunning(cmd)]
if md5 is not None:
actions.append((check_hash, [target_fn, md5]))
return {
'name': name,
'actions': actions,
'targets': [target_fn],
'clean': [clean_targets],
'uptodate': [True]
}
def get_download_and_untar_task(url, target_dir, label=None):
'''Create a doit task to download a file and untar it in the
given directory.
Args:
url (str): URL to download.
target_dir (str: Directory to put the untarred folder in.
label (str): Optional label to resolve doit name conflicts when putting
multiple results in the same folder.
Returns:
dict: doit task.
'''
if label is None:
label = os.path.basename(url)
cmd1 = 'mkdir -p {target_dir}; curl {url} | tar -xz -C {target_dir}'.format(
**locals())
name = 'download_and_untar:{0}-{1}'.format(os.path.basename(target_dir),
label)
done = os.path.join(target_dir, name) + '.done'
cmd2 = 'touch {done}'.format(done=done)
return {
'name': name,
'actions': [LongRunning(cmd1), cmd2],
'targets': [done],
'clean': [(clean_folder, [target_dir])],
'uptodate': [True]
}
def get_untargz_task(archive_fn, target_dir, label=None):
'''Create a doit task to untar and gunip a *.tar.gz archive.
Args:
archive_fn (str): The .tar.gz file.
target_dir (str): The folder to untar into.
label (str): Optional label to resolve doit task name conflicts.
Returns:
dict: doit task.
'''
if label is None:
label = os.path.basename(url)
cmd = 'tar -xzf -C {target_dir} {archive_fn}'.format(**locals())
name = 'untargz:{0}-{1}'.format(os.path.basename(target_dir), label)
done = os.path.join(target_dir, name) + '.done'
touch = 'touch {done}'.format(done=done)
return {
'name': name,
'actions':
['mkdir -p {0}'.format(target_dir),
LongRunning(cmd), touch],
'targets': [done],
'clean': [(clean_folder, [target_dir])],
'uptodate': [True]
}
def task_perf_local():
"""
run locustio performance tests on local deployment
"""
FLASK_PORT = 5000
ENVS = envs(
LOCAL_FLASK_PORT=FLASK_PORT,
AWS_ACCESS_KEY_ID="fake-id",
AWS_SECRET_ACCESS_KEY="fake-key",
PYTHONPATH=".",
)
cmd = f"env {ENVS} {PYTHON3} src/sub/app.py" # FIXME: should work on hub too...
return {
"basename":
"perf-local",
"task_dep": ["check", "venv"],
"actions": [
f"{PYTHON3} -m setup develop",
"echo $PATH",
LongRunning(
f"nohup env {envs} {PYTHON3} src/sub/app.py > /dev/null &"
), # FIXME: same as above
f"cd src/sub/tests/performance && locust -f locustfile.py --host=http://localhost:{FLASK_PORT}", # FIXME: same
],
}
def task_lint():
cmd = [
'pylint --rcfile=setup.cfg **/*.py', 'flake8',
'bandit -r --ini setup.cfg'
]
for c in cmd:
yield {'name': c.split()[0], 'actions': [LongRunning(c)]}
def task_reasoning():
"""Reads the RDF file with the data and starts the RDFox endpoint."""
cmd = [rdfox_path, "sandbox", dir_path, "exec scripts/reasoning/master"]
return {
'file_dep': reasoning_input + ['scripts/reasoning/master.rdfox'],
"uptodate": [False],
'actions': [LongRunning(cmd, shell=False)],
}
def task_launch():
"""launch the game"""
cmd = "scopes ./src/boot.sc"
return {
'actions': [LongRunning(cmd)],
'file_dep': runtime_targets,
'uptodate': [False]
}
def task_dynalite():
'''
dynalite db for testing and local runs: start, stop
'''
cmd = f'{DYNALITE} --port {CFG.DYNALITE_PORT}'
msg = f'dynalite server started on {CFG.DYNALITE_PORT} logging to {CFG.DYNALITE_FILE}'
def running():
mutex.acquire()
pid = None
try:
pid = call(f'lsof -i:{CFG.DYNALITE_PORT} -t')[1].strip()
except CalledProcessError:
return None
finally:
mutex.release()
mutex.acquire()
try:
args = call(f'ps -p {pid} -o args=')[1].strip()
except CalledProcessError:
return None
finally:
mutex.release()
if cmd in args:
return pid
return None
yield {
'name': 'stop',
'task_dep': [
'check',
'yarn',
],
'actions': [
f'kill {running()}',
],
'uptodate': [lambda: running() is None],
}
yield {
'name':
'start',
'task_dep': [
'check',
'yarn',
'dynalite:stop',
],
'actions': [
LongRunning(f'nohup {cmd} > {CFG.DYNALITE_FILE} &'),
f'echo "{msg}"',
],
'uptodate': [
lambda: running(),
],
}
def _nblint(nb, nb_ok):
return _ok(
dict(
name=f"""nblint:{nb.stem}""",
file_dep=[P.YARN_INTEGRITY, P.OK_ENV["qa"], nb],
actions=[
LongRunning([*P.APR_QA, *P.PYM, "_scripts.nblint", nb],
shell=False)
],
),
nb_ok,
)
def task_watch_docs():
"""continuously rebuild the docs on change"""
yield dict(
uptodate=[lambda: False],
name="sphinx-autobuild",
file_dep=[P.DOCS_BUILDINFO, *P.ALL_MD, P.OK_DOCS_PIP_INSTALL],
actions=[
LongRunning(
[*P.APR_DOCS, "sphinx-autobuild", P.DOCS, P.DOCS_BUILD], shell=False
)
],
)
def task_dynalite():
'''
dynalite db for testing and local runs: start, stop
'''
cmd = f'{DYNALITE} --port {CFG.DYNALITE_PORT}'
msg = f'dyanlite server started on {CFG.DYNALITE_PORT} logging to {CFG.DYNALITE_FILE}'
def running():
pid = None
try:
pid = call(f'lsof -i:{CFG.DYNALITE_PORT} -t')[1].strip()
except CalledProcessError:
return None
try:
args = call(f'ps -p {pid} -o args=')[1].strip()
except CalledProcessError:
return None
if f'{cmd}' in args:
return pid
return None
pid = running()
yield {
'name': 'stop',
'task_dep': [
'check',
'npm',
],
'actions': [
f'kill {pid}',
],
'uptodate': [
lambda: pid is None
],
}
yield {
'name': 'start',
'task_dep': [
'check',
'npm',
],
'actions': [
LongRunning(f'nohup {cmd} > {CFG.DYNALITE_FILE} &'),
f'echo "{msg}"',
],
'uptodate': [
lambda: pid,
],
}
def task_perf():
'''
run locustio performance tests
'''
ID = 'fake-id'
KEY = 'fake-key'
PP = '.'
FLASK_PORT = 5000
cmd = f'env {envs(LOCAL_FLASK_PORT=FLASK_PORT, AWS_ACCESS_KEY_ID=ID, AWS_SECRET_ACCESS_KEY=KEY, PYTHONPATH=PP)} {PYTHON3} subhub/app.py'
return {
'task_dep': ['check', 'stripe', 'venv', 'dynalite:start'],
'actions': [
f'{PYTHON3} -m setup develop', 'echo $PATH',
LongRunning(f'nohup {cmd} > /dev/null &'),
f'cd subhub/tests/performance && locust -f locustfile.py --host=http://localhost:{FLASK_PORT}'
]
}
def lastal_task(query,
db,
out_fn,
cutoff=0.00001,
n_threads=1,
translate=False,
cfg=LASTAL_CFG,
pbs=False):
'''Create a pydoit task to run lastal
Args:
query (str): The file with the query sequences.
db (str): The database file prefix.
out_fn (str): Destination file for alignments.
translate (bool): True if query is a nucleotide FASTA.
n_threads (int): Number of threads to run with.
cfg (dict): Config, must contain key params holding str.
Returns:
dict: A pydoit task.
'''
lastal_exc = which('lastal')
parallel_exc = which('parallel-fasta')
params = cfg['params']
lastal_cmd = [lastal_exc]
if translate:
lastal_cmd.append('-F' + str(cfg['frameshift']))
if cutoff is not None:
cutoff = round(1.0 / cutoff)
lastal_cmd.append('-D' + str(cutoff))
lastal_cmd.append(db)
lastal_cmd = ' '.join(lastal_cmd)
cmd = parallel_fasta(query, out_fn, lastal_cmd, n_threads, pbs=pbs)
name = 'lastal:{0}'.format(os.path.join(out_fn))
return {
'name': name,
'title': title,
'actions': [LongRunning(cmd)],
'targets': [out_fn],
'file_dep': [query, db + '.prj'],
'clean': [clean_targets]
}
def get_download_and_gunzip_task(url, target_fn):
'''Create a doit task which downloads and gunzips a file.
Args:
url (str): URL to download.
target_fn (str): Target file for the download.
Returns:
dict: doit task.
'''
cmd = 'curl {url} | gunzip -c > {target_fn}'.format(**locals())
name = 'download_and_gunzip:{0}'.format(os.path.basename(target_fn))
return {'name': name,
'actions': [LongRunning(cmd)],
'targets': [target_fn],
'clean': [clean_targets],
'uptodate': [True]}
def get_gunzip_task(archive_fn, target_fn):
'''Create a doit task to gunzip a gzip archive.
Args:
archive_fn (str): The gzip file.
target_fn (str): Output filename.
Returns:
dict: doit task.
'''
name = 'gunzip:{0}'.format(os.path.basename(target_fn))
cmd = 'gunzip -c {archive_fn} > {target_fn}'.format(**locals())
return {'name': name,
'actions': [LongRunning(cmd)],
'file_dep': [archive_fn],
'targets': [target_fn],
'clean': [clean_targets],
'uptodate': [True]}
def task_perf_local():
'''
run locustio performance tests on local deployment
'''
FLASK_PORT = 5000
ENVS = envs(LOCAL_FLASK_PORT=FLASK_PORT,
AWS_ACCESS_KEY_ID='fake-id',
AWS_SECRET_ACCESS_KEY='fake-key',
PYTHONPATH='.')
cmd = f'env {ENVS} {PYTHON3} subhub/app.py'
return {
'basename':
'perf-local',
'task_dep': ['check', 'stripe', 'venv', 'dynalite:start'],
'actions': [
f'{PYTHON3} -m setup develop', 'echo $PATH',
LongRunning(
f'nohup env {envs} {PYTHON3} subhub/app.py > /dev/null &'),
f'cd subhub/tests/performance && locust -f locustfile.py --host=http://localhost:{FLASK_PORT}'
]
}
def task_top():
cmd = "top"
return {'actions': [LongRunning(cmd)],}
def task_up():
cmd = "sudo " + DOCKER + " up --remove-orphans"
return {"actions": [LongRunning(cmd)], "verbosity": VERBOSITY_LEVEL}
def __init__(self, cfg):
super(CodeTaskClean, self).__init__('do_clean', cfg)
self.actions = [(self.save_timestamp,),
LongRunning(self.format_command(self.gf("clean")),
cwd=self.root_dir, save_out='cleaned')]
self.uptodate = [run_once]
def task_runserver():
""" django runserver """
cmd = 'python manage.py runserver'
return {'actions': [LongRunning(cmd)]}