def process(paths2process, db, wait=WAIT_TIME, logdir='log'): cmd = 'ls -l {0}' # if paths2process and # time.time() - os.path.getmtime(paths2process[0]) > WAIT_TIME: processed = [] for path, mod_time in paths2process.items(): if time.time() - mod_time > wait: #process_me = paths2process.popleft().decode('utf-8') process_me = path cmd_ = cmd.format(process_me) process_dict = {'input_path': process_me, 'accession_number': op.basename(process_me)} print("Time to process {0}".format(process_me)) stdout, run_dict = run_heudiconv(cmd_) process_dict.update(run_dict) db.insert(process_dict) # save log logdir = localpath(logdir) log = logdir.join(process_dict['accession_number'] + '.log') log.write(stdout) # if we processed it, or it failed, # we need to remove it to avoid running it again processed.append(path) for processed_path in processed: del paths2process[processed_path]
def process(paths2process, db, wait=WAIT_TIME, logdir='log'): cmd = 'ls -l {0}' # if paths2process and # time.time() - os.path.getmtime(paths2process[0]) > WAIT_TIME: processed = [] for path, mod_time in paths2process.items(): if time.time() - mod_time > wait: #process_me = paths2process.popleft().decode('utf-8') process_me = path cmd_ = cmd.format(process_me) process_dict = { 'input_path': process_me, 'accession_number': op.basename(process_me) } print("Time to process {0}".format(process_me)) stdout, run_dict = run_heudiconv(cmd_) process_dict.update(run_dict) db.insert(process_dict) # save log logdir = localpath(logdir) log = logdir.join(process_dict['accession_number'] + '.log') log.write(stdout) # if we processed it, or it failed, # we need to remove it to avoid running it again processed.append(path) for processed_path in processed: del paths2process[processed_path]
import fabric.state from fabric.decorators import task from fabric.contrib.files import upload_template from fabric.context_managers import settings, hide from fabric.api import sudo, run, env, execute, put, reboot from . import help # noqa from .utils import tobool fabric.state.output["running"] = False env.output_prefix = False TEMPLATES = localpath(__file__).new(basename="templates") def check_docker(*args, **kwargs): with settings(warn_only=True), hide("stdout", "running", "warnings"): out = run("which docker") if out == "": install_docker() def check_valid_os(*args, **kwargs): with settings(warn_only=True), hide("stdout", "running", "warnings"): out = run("which apt-get") if out == "": raise StandardError("Only Debian/Ubuntu are currently supported. Sorry.")
from py.path import local as localpath import fabric.state from fabric.decorators import task from fabric.contrib.files import upload_template from fabric.context_managers import settings, hide from fabric.api import sudo, run, env, execute, put, reboot from . import help # noqa from .utils import tobool fabric.state.output['running'] = False env.output_prefix = False TEMPLATES = localpath(__file__).new(basename="templates") def check_docker(*args, **kwargs): with settings(warn_only=True), hide('stdout', 'running', 'warnings'): out = run('which docker') if out == '': install_docker() def check_valid_os(*args, **kwargs): with settings(warn_only=True), hide('stdout', 'running', 'warnings'): out = run('which apt-get') if out == '': raise StandardError( 'Only Debian/Ubuntu are currently supported. Sorry.')
class Compile(Task): name = "test" def __init__(self, *args, **kwargs): super(Compile, self).__init__(*args, **kwargs) self.options = kwargs.get("options", {}) def run(self): return execute(compile, **self.options) p = localpath() for target in p.join("targets").listdir("*.py"): name = target.purebasename name = name.split("_", 1)[1] options = { "tests": "no", "target": str(target), "output": str(p.join("build", name)) } task = Compile(name=name, options=options) setattr(task, "__doc__", "Compile {0:s} target".format(name)) globals()[name] = task