def __init__(self, requests, response, cores=None, loglevel=logging.INFO): """Initialize a new grid master with the request and response queues and optionally the number of cores or slots available. If no cores are specified, the number of cores of the current machine are used """ #: the queue this master receives requests on self.requests = requests #: the queue this master sends responses to self.response = response #: currently queued jobs self.queued = {} #: currently running jobs self.running = {} #: number of currently available slots self.slots_available = multiprocessing.cpu_count() \ if not cores else cores #: number of totally available slots self.slots_total = self.slots_available #: the master logger self.log = getLogger("jip.grids.Master") #: self wait mode self.wait_mode = False #: current id self._current_id = 0 self.log.setLevel(loglevel) self.log.info("Master | Initialized with %d available slots", self.slots_available)
def __init__(self, _start=True, _remote_ids=False, cores=None, log_level=logging.INFO): self._current_id = 0 self.log = getLogger("jip.grids.LocalCluster") self.master_requests = None self.master_response = None self.master_process = None self._current_id self._remote_ids = _remote_ids self.cores = cores self.loglevel = log_level # set log level self.log.setLevel(log_level) # start the mater process if _start: self.start()
def _execute_job(requests, job_id, job): """Local grid executor method that takes a ``_Job`` instance and runs its command in a subprocess. :param request: the request queue to send completion state back to the master :param job_id: the local job id :param job: instance of local job """ log = getLogger("jip.grids.Executor") process = None # setup local signal handling def handle_term(sig, frame): log.error("Exec | Received termination signal") _terminate_process(process, log) requests.put(['DONE', job_id, 1]) sys.exit(1) signal.signal(signal.SIGTERM, handle_term) log.info("Exec | Start job %s", job_id) result = -1 try: cwd = job.working_directory stdout = open(job.stdout, 'w') stderr = open(job.stderr, 'w') cmd = job.cmd process = subprocess.Popen( "exec " + cmd, stdout=stdout, stderr=stderr, shell=True, cwd=cwd ) result = process.wait() except Exception as err: log.error("Exec | Error : %s", err, exc_info=True) requests.put(['FAILED', job_id, str(err)]) log.info("Exec | Job finished %s with %d", job_id, result) requests.put(['DONE', job_id, result])
def _execute_job(requests, job_id, job): """Local grid executor method that takes a ``_Job`` instance and runs its command in a subprocess. :param request: the request queue to send completion state back to the master :param job_id: the local job id :param job: instance of local job """ log = getLogger("jip.grids.Executor") process = None # setup local signal handling def handle_term(sig, frame): log.error("Exec | Received termination signal") _terminate_process(process, log) requests.put(['DONE', job_id, 1]) sys.exit(1) signal.signal(signal.SIGTERM, handle_term) log.info("Exec | Start job %s", job_id) result = -1 try: cwd = job.working_directory stdout = open(job.stdout, 'w') stderr = open(job.stderr, 'w') cmd = job.cmd process = subprocess.Popen("exec " + cmd, stdout=stdout, stderr=stderr, shell=True, cwd=cwd) result = process.wait() except Exception as err: log.error("Exec | Error : %s", err, exc_info=True) requests.put(['FAILED', job_id, str(err)]) log.info("Exec | Job finished %s with %d", job_id, result) requests.put(['DONE', job_id, result])
import os from jinja2 import Environment, Undefined, contextfilter from jinja2.exceptions import TemplateSyntaxError from jip.logger import getLogger from jip.options import Option from jip.six import string_types, iteritems # contains global variable that # will be added to any render context # if they do not exists in the local context global_context = None #: the jinja2 environment environment = None log = getLogger('jip.templates') class RenderError(Exception): """Exception raised in case a tempalte could not be rendered properly """ def __init__(self, template, message, line=None): """Create a new render error instance :param template: the template string :param message: the error message :param line: optional line number """ self.template = template self.message = message self.line = line
import sys from sqlalchemy import Column, Integer, String, DateTime, \ ForeignKey, Table, orm from sqlalchemy import Text, Boolean, PickleType, bindparam, select, or_, and_ from sqlalchemy.orm import relationship, deferred, backref from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.exc import OperationalError from sqlalchemy.orm.exc import NoResultFound from jip.logger import getLogger from jip.tempfiles import create_temp_file from jip.six import string_types, StringIO, iteritems from jip.six.moves.urllib.parse import urlsplit, urlunsplit log = getLogger('jip.db') # global instances engine = None Session = None db_path = None db_in_memory = False global_session = None Base = declarative_base() #: Job is submitted but on hold STATE_HOLD = "Hold" #: Job is submitted to the compute cluster and is queued for execution
<args> optional script argument Other Options: -h --help Show this help message """ import sys import jip import jip.profiles from . import parse_args, show_dry, show_commands, colorize, RED, \ YELLOW import jip.jobs from jip.logger import getLogger log = getLogger("jip.cli.jip_submit") def main(argv=None): args = parse_args(__doc__, argv=argv) script_file = args["<tool>"] script_args = args["<args>"] try: script = jip.find(script_file) except LookupError, e: print >> sys.stderr, str(e) sys.exit(1) # load profile profile = jip.profiles.get( name='default' if not args['--profile'] else args['--profile'])
:py:class:`jip.executils.DispatcherNode` instances. The dispatcher nodes are executable units that can be started with their `run` methods. They will run asynchroniously and you have to use the nodes `wait` method to wait for termination. """ import sys import jip.db from jip.logger import getLogger import jip.cluster import jip.jobs import jip.profiler from jip.six import itervalues, iteritems log = getLogger('jip.executils') def create_dispatcher_graph(job, _nodes=None): """Create a dispatcher graph for a given job. If the job does not have any pipe targets, a list with a single dispatcher node is returned, otherwise the dispatching graph is created from all the pipe target job. :param job: the job :type: `jip.db.Job` :returns: list of dispatcher nodes :rtype: list of `jip.executils.DispatcherNode` instances """ # collect all jobs that are part # of this graph if len(job.pipe_to) == 0 and _nodes is None:
import os import sys import json from . import parse_args import jip import jip.jobs import jip.tools import jip.options import jip.profiles import jip.pipelines from jip.six import iteritems from jip.logger import getLogger log = getLogger('jip.cli.jip_specs') ################################################## # This is a bit of a hack :) # We want the output to be printed in a specific # order and have this working properly with 2.6 as # well (no OrderedDict in 2.6). Therefore we subclass # dictionary and ensure the items are yield in the order # # NOTE that you have to add keys manually here!! ################################################## class sorted_dict(dict): def items(self): ks = self.keys() sorted_keys = [
-l, --loglevel <level> The log level. On of DEBUG, INFO, WARN, ERROR [default: INFO] Other Options: -h --help Show this help message """ import logging from jip.logger import getLogger from . import parse_args import jip.grids import sys import jip.db log = getLogger("jip.cli.jip_server") def main(): args = parse_args(__doc__, options_first=True) try: import zmq except: print >>sys.stderr, """ Unable to import the python ZeroMQ binding. Please make sure that zeromq is installed on your system. You can install zeroMQ using pip: pip intall pyzmq
-c, --cmd <cmd> The bash command line that will be wrapped -h --help Show this help message """ from __future__ import print_function import jip import jip.cluster import jip.cli import jip.profiles import jip.jobs from jip.logger import getLogger from . import parse_args, colorize, YELLOW, RED import sys log = getLogger("jip.cli.jip_pipe") def main(): args = parse_args(__doc__, options_first=False) pipeline = jip.Pipeline() if not args['--cmd']: args['--cmd'] = "\n".join(sys.stdin.readlines()) if not args['--cmd']: print("No Command specified!", file=sys.stderr) sys.exit(1) @jip.pipeline() def embedded_pipeline(): """Embedded pipeline to run a custom pipeline script
Other Options: -h --help Show this help message """ from __future__ import print_function import sys from . import parse_args, dry, colorize, YELLOW, GREEN, RED, BLUE import jip import jip.jobs import jip.profiles from jip.logger import getLogger from datetime import datetime, timedelta log = getLogger('jip.cli.jip_run') def main(argv=None): args = parse_args(__doc__, argv=argv) script_file = args["<tool>"] script_args = args["<args>"] try: script = jip.find(script_file, is_pipeline=args['--pipeline']) except LookupError as e: print(str(e), file=sys.stderr) sys.exit(1) if args['--dry'] or args['--show']: dry(script, script_args, dry=args['--dry'], show=args['--show']) return
-l, --loglevel <level> The log level. On of DEBUG, INFO, WARN, ERROR [default: INFO] Other Options: -h --help Show this help message """ import logging from jip.logger import getLogger from . import parse_args import jip.grids import sys import jip.db log = getLogger("jip.cli.jip_server") def main(): args = parse_args(__doc__, options_first=True) try: import zmq except: print >> sys.stderr, """ Unable to import the python ZeroMQ binding. Please make sure that zeromq is installed on your system. You can install zeroMQ using pip: pip intall pyzmq
--force Force execution/submission --with-profiler execute the run with a profiler -c, --cmd <cmd> The bash command line that will be wrapped -h --help Show this help message """ import jip import jip.cluster import jip.cli import jip.profiles from jip.logger import getLogger from . import parse_args, colorize, YELLOW, RED import sys log = getLogger("jip.cli.jip_pipe") def main(): args = parse_args(__doc__, options_first=False) pipeline = jip.Pipeline() if not args['--cmd']: args['--cmd'] = "\n".join(sys.stdin.readlines()) cmd = args['--cmd'] if not args['--cmd']: print >>sys.stderr, "No Command specified!" sys.exit(1) @jip.pipeline() def embedded_pipeline(): """Embedded pipeline to run a custom pipeline script
--force Force execution/submission --with-profiler execute the run with a profiler -c, --cmd <cmd> The bash command line that will be wrapped -h --help Show this help message """ import jip import jip.cluster import jip.cli import jip.profiles from jip.logger import getLogger from . import parse_args, colorize, YELLOW, RED import sys log = getLogger("jip.cli.jip_bash") def main(): args = parse_args(__doc__, options_first=False) pipeline = jip.Pipeline() bash = pipeline.job( args['--name'] if args['--name'] else 'bash' ).run('bash') if not args['--cmd']: args['--cmd'] = "\n".join(sys.stdin.readlines()) bash.input = [sys.stdin if a == 'stdin' else a for a in args['--input']] bash.output = [sys.stdout if a == 'stdout' else a for a in args['--output']]
<tool> the tool that will be executed <args> optional script argument Other Options: -h --help Show this help message """ import sys from . import parse_args, dry, colorize, YELLOW, GREEN, RED, BLUE import jip import jip.jobs from jip.logger import getLogger from datetime import datetime, timedelta log = getLogger('jip.cli.jip_run') def main(argv=None): args = parse_args(__doc__, argv=argv) script_file = args["<tool>"] script_args = args["<args>"] try: script = jip.find(script_file, is_pipeline=args['--pipeline']) except LookupError, e: print >>sys.stderr, str(e) sys.exit(1) if args['--dry'] or args['--show']: dry(script, script_args, dry=args['--dry'], show=args['--show']) return
jip-check [--help|-h] [-d <db>] Options: -d, --db <db> the database source that will be used to find the job Other Options: -h --help Show this help message """ from jip.logger import getLogger import jip.db import jip.cluster import jip.executils from . import parse_args log = getLogger("jip.cli.jip_check") def main(): args = parse_args(__doc__, options_first=True) # get the cluster cluster = jip.cluster.get() # init the database and a session jip.db.init(path=args['--db']) session = jip.db.create_session() # get the job list from the cluster cluster_jobs = set(cluster.list()) # get all jobs that are queued or running query = session.query(jip.db.Job).filter( jip.db.Job.state.in_([jip.db.STATE_QUEUED, jip.db.STATE_RUNNING]) )
import os import subprocess import sys from sqlalchemy import Column, Integer, String, DateTime, \ ForeignKey, Table, orm from sqlalchemy import Text, Boolean, PickleType, bindparam, select, or_, and_ from sqlalchemy.orm import relationship, deferred, backref from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.exc import OperationalError from sqlalchemy.orm.exc import NoResultFound from jip.logger import getLogger from jip.tempfiles import create_temp_file log = getLogger('jip.db') # global instances engine = None Session = None db_path = None db_in_memory = False global_session = None Base = declarative_base() #: Job is submitted but on hold STATE_HOLD = "Hold" #: Job is submitted to the compute cluster and is queued for execution
-d, --db <db> the database source that will be used to find the job <id> the job id of the job that will be executed Other Options: -h --help Show this help message """ from jip.logger import getLogger import jip.jobs import jip.db from jip.six import iteritems from . import parse_args import sys import os log = getLogger("jip.cli.jip_exec") def main(): log.debug("job execution python path: %s", sys.path) args = parse_args(__doc__, options_first=True) try: log.info("Starting job with id %s stored in %s", args['<id>'], args['--db']) jip.db.init(path=args['--db']) job = jip.db.get(args['<id>']) if not job: log.error("Requested job with id %s not found!", args['<id>']) sys.exit(1) if job.state != jip.db.STATE_QUEUED: log.warn("Job does not come from queued state! Stoping execution")
jip-check [--help|-h] [-d <db>] Options: -d, --db <db> the database source that will be used to find the job Other Options: -h --help Show this help message """ from jip.logger import getLogger import jip.db import jip.cluster import jip.executils from . import parse_args log = getLogger("jip.cli.jip_check") def main(): args = parse_args(__doc__, options_first=True) # get the cluster cluster = jip.cluster.get() # init the database and a session jip.db.init(path=args['--db']) session = jip.db.create_session() # get the job list from the cluster cluster_jobs = set(cluster.list()) # get all jobs that are queued or running query = session.query(jip.db.Job).filter( jip.db.Job.state.in_([jip.db.STATE_QUEUED, jip.db.STATE_RUNNING])) for job in query:
def __init__(self): self.log = getLogger("jip.grids.JIP") cfg = jip.config.get('jip_grid', {}) self.host = cfg.get('host', '127.0.0.1') self.port = cfg.get('port', '5556') self.log = getLogger("jip.cluster.JIP")
""" import collections import os import re from subprocess import Popen, PIPE import multiprocessing import jip from jip.logger import getLogger #: internal cache to store the cluster instances _cluster_cache = {} #: the logger instance log = getLogger('jip.cluster') class SubmissionError(Exception): """This exception is raised if a job submission failed.""" pass class ClusterImplementationError(Exception): """Exception raised in case the cluster class could not be loaded""" pass class Cluster(object): """Base class for cluster integrations.
-n, --lines <lines> Show the given number of lines from the head/tail of the file [Default: 10] --head Show the head instead of the tail -h --help Show this help message """ from . import colorize, GREEN, RED, BLUE, STATE_COLORS import jip.cluster from . import parse_args, parse_job_ids from subprocess import Popen from os.path import exists from jip.logger import getLogger log = getLogger("jip.cli.jip_logs") def main(): args = parse_args(__doc__, options_first=False) job_ids, cluster_ids = parse_job_ids(args) if not job_ids and not cluster_ids: print __doc__.strip("\n") return jobs = jip.db.query(job_ids=job_ids, cluster_ids=cluster_ids, archived=None, fields=['stdout', 'stderr', 'job_id', 'id', 'name', 'state']) for job in jobs: cluster = jip.cluster.get() show_log(job, cluster, args)
""" import os import re from subprocess import Popen, PIPE import jip from jip.logger import getLogger from jip.six import long #: internal cache to store the cluster instances _cluster_cache = {} #: the logger instance log = getLogger('jip.cluster') class SubmissionError(Exception): """This exception is raised if a job submission failed.""" pass class ClusterImplementationError(Exception): """Exception raised in case the cluster class could not be loaded""" pass class ExecutableNotFoundError(Exception): def __init__(self, executable): self.executable = executable
function. The functions returns a sorted list of :py:class:`jip.executils.DispatcherNode` instances. The dispatcher nodes are executable units that can be started with their `run` methods. They will run asynchroniously and you have to use the nodes `wait` method to wait for termination. """ import sys import jip.db from jip.logger import getLogger import jip.cluster import jip.jobs import jip.profiler log = getLogger('jip.executils') def create_dispatcher_graph(job, _nodes=None): """Create a dispatcher graph for a given job. If the job does not have any pipe targets, a list with a single dispatcher node is returned, otherwise the dispatching graph is created from all the pipe target job. :param job: the job :type: `jip.db.Job` :returns: list of dispatcher nodes :rtype: list of `jip.executils.DispatcherNode` instances """ # collect all jobs that are part # of this graph if len(job.pipe_to) == 0 and _nodes is None:
Issue Tracker: https://github.com/thasso/pyjip/issues """ import os import sys import jip import jip.options import jip.tools import jip.cli import jip.cluster import jip.configuration import jip.templates from jip.logger import getLogger, log_level from jip.vendor.docopt import docopt log = getLogger('jip.cli.jip_main') def main(): try: jip.configuration.install_path = os.path.abspath( os.path.dirname(sys.argv[0])) except: pass try: _main() except jip.options.ParserException as err: log.debug("parser error: %s", str(err), exc_info=True) sys.stderr.write(str(err)) sys.exit(1)
Options: -d, --db <db> the database source that will be used to find the job <id> the job id of the job that will be executed Other Options: -h --help Show this help message """ from jip.logger import getLogger import jip.jobs import jip.db from . import parse_args import sys import os log = getLogger("jip.cli.jip_exec") def main(): log.debug("job execution python path: %s", sys.path) args = parse_args(__doc__, options_first=True) try: log.info("Starting job with id %s stored in %s", args['<id>'], args['--db']) jip.db.init(path=args['--db']) job = jip.db.get(args['<id>']) if not job: log.error("Requested job with id %s not found!", args['<id>']) sys.exit(1) if job.state != jip.db.STATE_QUEUED:
Issue Tracker: https://github.com/thasso/pyjip/issues """ import os import sys import jip import jip.options import jip.tools import jip.cli import jip.cluster import jip.configuration import jip.templates from jip.logger import getLogger, log_level from jip.vendor.docopt import docopt log = getLogger('jip.cli.jip_main') def main(): try: jip.configuration.install_path = os.path.abspath( os.path.dirname(sys.argv[0]) ) except: pass try: _main() except jip.options.ParserException as err: log.debug("parser error: %s", str(err), exc_info=True) sys.stderr.write(str(err))
-h --help Show this help message """ import os import sys import json from . import parse_args import jip import jip.jobs import jip.tools import jip.options import jip.profiles import jip.pipelines from jip.logger import getLogger log = getLogger('jip.cli.jip_specs') ################################################## # This is a bit of a hack :) # We want the output to be printed in a specific # order and have this working properly with 2.6 as # well (no OrderedDict in 2.6). Therefore we subclass # dictionary and ensure the items are yield in the order # # NOTE that you have to add keys manually here!! ################################################## class sorted_dict(dict): def items(self): ks = self.keys() sorted_keys = [
-c, --cmd <cmd> The bash command line that will be wrapped -h --help Show this help message """ from __future__ import print_function import jip import jip.jobs import jip.cluster import jip.cli import jip.profiles from jip.logger import getLogger from . import parse_args, colorize, YELLOW, RED import sys log = getLogger("jip.cli.jip_bash") def main(): args = parse_args(__doc__, options_first=False) pipeline = jip.Pipeline() bash = pipeline.job(args['--name'] if args['--name'] else 'bash').run( 'bash') if not args['--cmd']: args['--cmd'] = "\n".join(sys.stdin.readlines()) bash.input = [sys.stdin if a == 'stdin' else a for a in args['--input']] bash.output = [ sys.stdout if a == 'stdout' else a for a in args['--output'] ] bash.outfile = [a for a in args['--outfile']]
<args> optional script argument Other Options: -h --help Show this help message """ import sys import jip import jip.profiles from . import parse_args, show_dry, show_commands, colorize, RED, \ YELLOW import jip.jobs from jip.logger import getLogger log = getLogger("jip.cli.jip_submit") def main(argv=None): args = parse_args(__doc__, argv=argv) script_file = args["<tool>"] script_args = args["<args>"] try: script = jip.find(script_file) except LookupError, e: print >>sys.stderr, str(e) sys.exit(1) # load profile profile = jip.profiles.get(name='default' if not args['--profile']
thought the :py:func:`render_template` function. """ import os from jinja2 import Environment, Undefined, contextfilter from jip.logger import getLogger from jip.options import Option # contains global variable that # will be added to any render context # if they do not exists in the local context global_context = None #: the jinja2 environment environment = None log = getLogger('jip.templates') def set_global_context(global_ctx): global global_context if not global_ctx: raise global_context = global_ctx class JipUndefined(Undefined): """Custom undefined implementation that does not modify unknown variables """ def __unicode__(self): log.info("Unknown context variable: %s", self._undefined_name)
[Default: 10] --head Show the head instead of the tail -h --help Show this help message """ from . import colorize, GREEN, RED, BLUE, STATE_COLORS import jip.cluster from . import parse_args, parse_job_ids from subprocess import Popen from os.path import exists import jip.db from jip.logger import getLogger log = getLogger("jip.cli.jip_logs") def main(): args = parse_args(__doc__, options_first=False) job_ids, cluster_ids = parse_job_ids(args) if not job_ids and not cluster_ids: print(__doc__.strip("\n")) return jobs = jip.db.query(job_ids=job_ids, cluster_ids=cluster_ids, archived=None, fields=['stdout', 'stderr', 'job_id', 'id', 'name', 'state']) for job in jobs: cluster = jip.cluster.get() show_log(job, cluster, args)