Exemple #1
0
    def __init__(self, requests, response, cores=None, loglevel=logging.INFO):
        """Initialize a new grid master with the request and response
        queues and optionally the number of cores or slots available.
        If no cores are specified, the number of cores of the current
        machine are used
        """
        #: the queue this master receives requests on
        self.requests = requests
        #: the queue this master sends responses to
        self.response = response
        #: currently queued jobs
        self.queued = {}
        #: currently running jobs
        self.running = {}
        #: number of currently available slots
        self.slots_available = multiprocessing.cpu_count() \
            if not cores else cores
        #: number of totally available slots
        self.slots_total = self.slots_available
        #: the master logger
        self.log = getLogger("jip.grids.Master")
        #: self wait mode
        self.wait_mode = False
        #: current id
        self._current_id = 0

        self.log.setLevel(loglevel)
        self.log.info("Master | Initialized with %d available slots",
                      self.slots_available)
Exemple #2
0
    def __init__(self, requests, response, cores=None, loglevel=logging.INFO):
        """Initialize a new grid master with the request and response
        queues and optionally the number of cores or slots available.
        If no cores are specified, the number of cores of the current
        machine are used
        """
        #: the queue this master receives requests on
        self.requests = requests
        #: the queue this master sends responses to
        self.response = response
        #: currently queued jobs
        self.queued = {}
        #: currently running jobs
        self.running = {}
        #: number of currently available slots
        self.slots_available = multiprocessing.cpu_count() \
            if not cores else cores
        #: number of totally available slots
        self.slots_total = self.slots_available
        #: the master logger
        self.log = getLogger("jip.grids.Master")
        #: self wait mode
        self.wait_mode = False
        #: current id
        self._current_id = 0

        self.log.setLevel(loglevel)
        self.log.info("Master | Initialized with %d available slots",
                      self.slots_available)
Exemple #3
0
 def __init__(self, _start=True,
              _remote_ids=False,
              cores=None, log_level=logging.INFO):
     self._current_id = 0
     self.log = getLogger("jip.grids.LocalCluster")
     self.master_requests = None
     self.master_response = None
     self.master_process = None
     self._current_id
     self._remote_ids = _remote_ids
     self.cores = cores
     self.loglevel = log_level
     # set log level
     self.log.setLevel(log_level)
     # start the mater process
     if _start:
         self.start()
Exemple #4
0
 def __init__(self,
              _start=True,
              _remote_ids=False,
              cores=None,
              log_level=logging.INFO):
     self._current_id = 0
     self.log = getLogger("jip.grids.LocalCluster")
     self.master_requests = None
     self.master_response = None
     self.master_process = None
     self._current_id
     self._remote_ids = _remote_ids
     self.cores = cores
     self.loglevel = log_level
     # set log level
     self.log.setLevel(log_level)
     # start the mater process
     if _start:
         self.start()
Exemple #5
0
def _execute_job(requests, job_id, job):
    """Local grid executor method that takes a ``_Job`` instance and
    runs its command in a subprocess.

    :param request: the request queue to send completion state back to
                    the master
    :param job_id: the local job id
    :param job: instance of local job
    """
    log = getLogger("jip.grids.Executor")
    process = None

    # setup local signal handling
    def handle_term(sig, frame):
        log.error("Exec | Received termination signal")
        _terminate_process(process, log)
        requests.put(['DONE', job_id, 1])
        sys.exit(1)

    signal.signal(signal.SIGTERM, handle_term)

    log.info("Exec | Start job %s", job_id)
    result = -1
    try:
        cwd = job.working_directory
        stdout = open(job.stdout, 'w')
        stderr = open(job.stderr, 'w')
        cmd = job.cmd
        process = subprocess.Popen(
            "exec " + cmd,
            stdout=stdout,
            stderr=stderr,
            shell=True,
            cwd=cwd
        )
        result = process.wait()
    except Exception as err:
        log.error("Exec | Error : %s", err, exc_info=True)
        requests.put(['FAILED', job_id, str(err)])
    log.info("Exec | Job finished %s with %d", job_id, result)
    requests.put(['DONE', job_id, result])
Exemple #6
0
def _execute_job(requests, job_id, job):
    """Local grid executor method that takes a ``_Job`` instance and
    runs its command in a subprocess.

    :param request: the request queue to send completion state back to
                    the master
    :param job_id: the local job id
    :param job: instance of local job
    """
    log = getLogger("jip.grids.Executor")
    process = None

    # setup local signal handling
    def handle_term(sig, frame):
        log.error("Exec | Received termination signal")
        _terminate_process(process, log)
        requests.put(['DONE', job_id, 1])
        sys.exit(1)

    signal.signal(signal.SIGTERM, handle_term)

    log.info("Exec | Start job %s", job_id)
    result = -1
    try:
        cwd = job.working_directory
        stdout = open(job.stdout, 'w')
        stderr = open(job.stderr, 'w')
        cmd = job.cmd
        process = subprocess.Popen("exec " + cmd,
                                   stdout=stdout,
                                   stderr=stderr,
                                   shell=True,
                                   cwd=cwd)
        result = process.wait()
    except Exception as err:
        log.error("Exec | Error : %s", err, exc_info=True)
        requests.put(['FAILED', job_id, str(err)])
    log.info("Exec | Job finished %s with %d", job_id, result)
    requests.put(['DONE', job_id, result])
Exemple #7
0
import os
from jinja2 import Environment, Undefined, contextfilter
from jinja2.exceptions import TemplateSyntaxError
from jip.logger import getLogger
from jip.options import Option
from jip.six import string_types, iteritems

# contains global variable that
# will be added to any render context
# if they do not exists in the local context
global_context = None

#: the jinja2 environment
environment = None

log = getLogger('jip.templates')


class RenderError(Exception):
    """Exception raised in case a tempalte could not be rendered properly
    """
    def __init__(self, template, message, line=None):
        """Create a new render error instance

        :param template: the template string
        :param message: the error message
        :param line: optional line number
        """
        self.template = template
        self.message = message
        self.line = line
Exemple #8
0
import sys

from sqlalchemy import Column, Integer, String, DateTime, \
    ForeignKey, Table, orm
from sqlalchemy import Text, Boolean, PickleType, bindparam, select, or_, and_
from sqlalchemy.orm import relationship, deferred, backref
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.exc import OperationalError
from sqlalchemy.orm.exc import NoResultFound

from jip.logger import getLogger
from jip.tempfiles import create_temp_file
from jip.six import string_types, StringIO, iteritems
from jip.six.moves.urllib.parse import urlsplit, urlunsplit

log = getLogger('jip.db')


# global instances
engine = None
Session = None
db_path = None
db_in_memory = False
global_session = None

Base = declarative_base()


#: Job is submitted but on hold
STATE_HOLD = "Hold"
#: Job is submitted to the compute cluster and is queued for execution
Exemple #9
0
  <args>                   optional script argument

Other Options:
    -h --help             Show this help message

"""
import sys

import jip
import jip.profiles
from . import parse_args, show_dry, show_commands, colorize, RED, \
    YELLOW
import jip.jobs
from jip.logger import getLogger

log = getLogger("jip.cli.jip_submit")


def main(argv=None):
    args = parse_args(__doc__, argv=argv)
    script_file = args["<tool>"]
    script_args = args["<args>"]
    try:
        script = jip.find(script_file)
    except LookupError, e:
        print >> sys.stderr, str(e)
        sys.exit(1)

    # load profile
    profile = jip.profiles.get(
        name='default' if not args['--profile'] else args['--profile'])
Exemple #10
0
:py:class:`jip.executils.DispatcherNode` instances. The dispatcher nodes are
executable units that can be started with their `run` methods. They will
run asynchroniously and you have to use the nodes `wait` method to wait
for termination.
"""
import sys

import jip.db
from jip.logger import getLogger
import jip.cluster
import jip.jobs
import jip.profiler

from jip.six import itervalues, iteritems

log = getLogger('jip.executils')


def create_dispatcher_graph(job, _nodes=None):
    """Create a dispatcher graph for a given job. If the job does not
    have any pipe targets, a list with a single dispatcher node is returned,
    otherwise the dispatching graph is created from all the pipe target job.

    :param job: the job
    :type: `jip.db.Job`
    :returns: list of dispatcher nodes
    :rtype: list of `jip.executils.DispatcherNode` instances
    """
    # collect all jobs that are part
    # of this graph
    if len(job.pipe_to) == 0 and _nodes is None:
Exemple #11
0
import os
import sys
import json

from . import parse_args
import jip
import jip.jobs
import jip.tools
import jip.options
import jip.profiles
import jip.pipelines
from jip.six import iteritems
from jip.logger import getLogger

log = getLogger('jip.cli.jip_specs')


##################################################
# This is a bit of a hack :)
# We want the output to be printed in a specific
# order and have this working properly with 2.6 as
# well (no OrderedDict in 2.6). Therefore we subclass
# dictionary and ensure the items are yield in the order
#
# NOTE that you have to add keys manually here!!
##################################################
class sorted_dict(dict):
    def items(self):
        ks = self.keys()
        sorted_keys = [
Exemple #12
0
    -l, --loglevel <level>   The log level. On of DEBUG, INFO, WARN, ERROR
                             [default: INFO]

Other Options:
    -h --help             Show this help message
"""

import logging

from jip.logger import getLogger
from . import parse_args
import jip.grids
import sys
import jip.db

log = getLogger("jip.cli.jip_server")


def main():
    args = parse_args(__doc__, options_first=True)

    try:
        import zmq
    except:
        print >>sys.stderr, """
Unable to import the python ZeroMQ binding.
Please make sure that zeromq is installed on your system.

You can install zeroMQ using pip:

    pip intall pyzmq
Exemple #13
0
    -c, --cmd <cmd>          The bash command line that will be wrapped
    -h --help                Show this help message

"""
from __future__ import print_function

import jip
import jip.cluster
import jip.cli
import jip.profiles
import jip.jobs
from jip.logger import getLogger
from . import parse_args, colorize, YELLOW, RED
import sys

log = getLogger("jip.cli.jip_pipe")


def main():
    args = parse_args(__doc__, options_first=False)
    pipeline = jip.Pipeline()
    if not args['--cmd']:
        args['--cmd'] = "\n".join(sys.stdin.readlines())
    if not args['--cmd']:
        print("No Command specified!", file=sys.stderr)
        sys.exit(1)

    @jip.pipeline()
    def embedded_pipeline():
        """Embedded pipeline to run a custom pipeline script
Exemple #14
0
Other Options:
    -h --help             Show this help message

"""
from __future__ import print_function

import sys

from . import parse_args, dry, colorize, YELLOW, GREEN, RED, BLUE
import jip
import jip.jobs
import jip.profiles
from jip.logger import getLogger
from datetime import datetime, timedelta

log = getLogger('jip.cli.jip_run')


def main(argv=None):
    args = parse_args(__doc__, argv=argv)
    script_file = args["<tool>"]
    script_args = args["<args>"]
    try:
        script = jip.find(script_file, is_pipeline=args['--pipeline'])
    except LookupError as e:
        print(str(e), file=sys.stderr)
        sys.exit(1)

    if args['--dry'] or args['--show']:
        dry(script, script_args, dry=args['--dry'], show=args['--show'])
        return
Exemple #15
0
    -l, --loglevel <level>   The log level. On of DEBUG, INFO, WARN, ERROR
                             [default: INFO]

Other Options:
    -h --help             Show this help message
"""

import logging

from jip.logger import getLogger
from . import parse_args
import jip.grids
import sys
import jip.db

log = getLogger("jip.cli.jip_server")


def main():
    args = parse_args(__doc__, options_first=True)

    try:
        import zmq
    except:
        print >> sys.stderr, """
Unable to import the python ZeroMQ binding.
Please make sure that zeromq is installed on your system.

You can install zeroMQ using pip:

    pip intall pyzmq
Exemple #16
0
    --force                  Force execution/submission
    --with-profiler          execute the run with a profiler
    -c, --cmd <cmd>          The bash command line that will be wrapped
    -h --help                Show this help message

"""
import jip
import jip.cluster
import jip.cli
import jip.profiles
from jip.logger import getLogger
from . import parse_args, colorize, YELLOW, RED
import sys


log = getLogger("jip.cli.jip_pipe")


def main():
    args = parse_args(__doc__, options_first=False)
    pipeline = jip.Pipeline()
    if not args['--cmd']:
        args['--cmd'] = "\n".join(sys.stdin.readlines())
    cmd = args['--cmd']
    if not args['--cmd']:
        print >>sys.stderr, "No Command specified!"
        sys.exit(1)

    @jip.pipeline()
    def embedded_pipeline():
        """Embedded pipeline to run a custom pipeline script
Exemple #17
0
    --force                  Force execution/submission
    --with-profiler          execute the run with a profiler
    -c, --cmd <cmd>          The bash command line that will be wrapped
    -h --help                Show this help message

"""
import jip
import jip.cluster
import jip.cli
import jip.profiles
from jip.logger import getLogger
from . import parse_args, colorize, YELLOW, RED
import sys


log = getLogger("jip.cli.jip_bash")


def main():
    args = parse_args(__doc__, options_first=False)
    pipeline = jip.Pipeline()
    bash = pipeline.job(
        args['--name'] if args['--name'] else 'bash'
    ).run('bash')
    if not args['--cmd']:
        args['--cmd'] = "\n".join(sys.stdin.readlines())

    bash.input = [sys.stdin if a == 'stdin' else a
                  for a in args['--input']]
    bash.output = [sys.stdout if a == 'stdout' else a
                   for a in args['--output']]
Exemple #18
0
  <tool>                   the tool that will be executed
  <args>                   optional script argument

Other Options:
    -h --help             Show this help message

"""
import sys

from . import parse_args, dry, colorize, YELLOW, GREEN, RED, BLUE
import jip
import jip.jobs
from jip.logger import getLogger
from datetime import datetime, timedelta

log = getLogger('jip.cli.jip_run')


def main(argv=None):
    args = parse_args(__doc__, argv=argv)
    script_file = args["<tool>"]
    script_args = args["<args>"]
    try:
        script = jip.find(script_file, is_pipeline=args['--pipeline'])
    except LookupError, e:
        print >>sys.stderr, str(e)
        sys.exit(1)

    if args['--dry'] or args['--show']:
        dry(script, script_args, dry=args['--dry'], show=args['--show'])
        return
Exemple #19
0
   jip-check [--help|-h] [-d <db>]

Options:
    -d, --db <db>  the database source that will be used to find the job

Other Options:
    -h --help             Show this help message
"""

from jip.logger import getLogger
import jip.db
import jip.cluster
import jip.executils
from . import parse_args

log = getLogger("jip.cli.jip_check")


def main():
    args = parse_args(__doc__, options_first=True)
    # get the cluster
    cluster = jip.cluster.get()
    # init the database and a session
    jip.db.init(path=args['--db'])
    session = jip.db.create_session()
    # get the job list from the cluster
    cluster_jobs = set(cluster.list())
    # get all jobs that are queued or running
    query = session.query(jip.db.Job).filter(
        jip.db.Job.state.in_([jip.db.STATE_QUEUED, jip.db.STATE_RUNNING])
    )
Exemple #20
0
import os
import subprocess
import sys

from sqlalchemy import Column, Integer, String, DateTime, \
    ForeignKey, Table, orm
from sqlalchemy import Text, Boolean, PickleType, bindparam, select, or_, and_
from sqlalchemy.orm import relationship, deferred, backref
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.exc import OperationalError
from sqlalchemy.orm.exc import NoResultFound

from jip.logger import getLogger
from jip.tempfiles import create_temp_file

log = getLogger('jip.db')


# global instances
engine = None
Session = None
db_path = None
db_in_memory = False
global_session = None

Base = declarative_base()


#: Job is submitted but on hold
STATE_HOLD = "Hold"
#: Job is submitted to the compute cluster and is queued for execution
Exemple #21
0
    -d, --db <db>  the database source that will be used to find the job
    <id>           the job id of the job that will be executed

Other Options:
    -h --help             Show this help message
"""

from jip.logger import getLogger
import jip.jobs
import jip.db
from jip.six import iteritems
from . import parse_args
import sys
import os

log = getLogger("jip.cli.jip_exec")


def main():
    log.debug("job execution python path: %s", sys.path)
    args = parse_args(__doc__, options_first=True)
    try:
        log.info("Starting job with id %s stored in %s", args['<id>'],
                 args['--db'])
        jip.db.init(path=args['--db'])
        job = jip.db.get(args['<id>'])
        if not job:
            log.error("Requested job with id %s not found!", args['<id>'])
            sys.exit(1)
        if job.state != jip.db.STATE_QUEUED:
            log.warn("Job does not come from queued state! Stoping execution")
Exemple #22
0
   jip-check [--help|-h] [-d <db>]

Options:
    -d, --db <db>  the database source that will be used to find the job

Other Options:
    -h --help             Show this help message
"""

from jip.logger import getLogger
import jip.db
import jip.cluster
import jip.executils
from . import parse_args

log = getLogger("jip.cli.jip_check")


def main():
    args = parse_args(__doc__, options_first=True)
    # get the cluster
    cluster = jip.cluster.get()
    # init the database and a session
    jip.db.init(path=args['--db'])
    session = jip.db.create_session()
    # get the job list from the cluster
    cluster_jobs = set(cluster.list())
    # get all jobs that are queued or running
    query = session.query(jip.db.Job).filter(
        jip.db.Job.state.in_([jip.db.STATE_QUEUED, jip.db.STATE_RUNNING]))
    for job in query:
Exemple #23
0
 def __init__(self):
     self.log = getLogger("jip.grids.JIP")
     cfg = jip.config.get('jip_grid', {})
     self.host = cfg.get('host', '127.0.0.1')
     self.port = cfg.get('port', '5556')
     self.log = getLogger("jip.cluster.JIP")
Exemple #24
0
 def __init__(self):
     self.log = getLogger("jip.grids.JIP")
     cfg = jip.config.get('jip_grid', {})
     self.host = cfg.get('host', '127.0.0.1')
     self.port = cfg.get('port', '5556')
     self.log = getLogger("jip.cluster.JIP")
Exemple #25
0
"""
import collections
import os
import re
from subprocess import Popen, PIPE
import multiprocessing

import jip
from jip.logger import getLogger


#: internal cache to store the cluster instances
_cluster_cache = {}

#: the logger instance
log = getLogger('jip.cluster')


class SubmissionError(Exception):
    """This exception is raised if a job submission failed."""
    pass


class ClusterImplementationError(Exception):
    """Exception raised in case the cluster class could not be loaded"""
    pass


class Cluster(object):
    """Base class for cluster integrations.
Exemple #26
0
    -n, --lines <lines>      Show the given number of lines from the head/tail
                             of the file
                             [Default: 10]
    --head                   Show the head instead of the tail
    -h --help                Show this help message
"""

from . import colorize, GREEN, RED, BLUE, STATE_COLORS
import jip.cluster
from . import parse_args, parse_job_ids

from subprocess import Popen
from os.path import exists
from jip.logger import getLogger

log = getLogger("jip.cli.jip_logs")


def main():
    args = parse_args(__doc__, options_first=False)
    job_ids, cluster_ids = parse_job_ids(args)
    if not job_ids and not cluster_ids:
        print __doc__.strip("\n")
        return
    jobs = jip.db.query(job_ids=job_ids, cluster_ids=cluster_ids,
                        archived=None, fields=['stdout', 'stderr',
                                               'job_id', 'id', 'name',
                                               'state'])
    for job in jobs:
        cluster = jip.cluster.get()
        show_log(job, cluster, args)
Exemple #27
0
"""
import os
import re
from subprocess import Popen, PIPE

import jip
from jip.logger import getLogger
from jip.six import long


#: internal cache to store the cluster instances
_cluster_cache = {}

#: the logger instance
log = getLogger('jip.cluster')


class SubmissionError(Exception):
    """This exception is raised if a job submission failed."""
    pass


class ClusterImplementationError(Exception):
    """Exception raised in case the cluster class could not be loaded"""
    pass


class ExecutableNotFoundError(Exception):
    def __init__(self, executable):
        self.executable = executable
Exemple #28
0
function. The functions returns a sorted list of
:py:class:`jip.executils.DispatcherNode` instances. The dispatcher nodes are
executable units that can be started with their `run` methods. They will
run asynchroniously and you have to use the nodes `wait` method to wait
for termination.
"""
import sys

import jip.db
from jip.logger import getLogger
import jip.cluster
import jip.jobs
import jip.profiler


log = getLogger('jip.executils')


def create_dispatcher_graph(job, _nodes=None):
    """Create a dispatcher graph for a given job. If the job does not
    have any pipe targets, a list with a single dispatcher node is returned,
    otherwise the dispatching graph is created from all the pipe target job.

    :param job: the job
    :type: `jip.db.Job`
    :returns: list of dispatcher nodes
    :rtype: list of `jip.executils.DispatcherNode` instances
    """
    # collect all jobs that are part
    # of this graph
    if len(job.pipe_to) == 0 and _nodes is None:
Exemple #29
0
Issue Tracker: https://github.com/thasso/pyjip/issues
"""
import os
import sys
import jip
import jip.options
import jip.tools
import jip.cli
import jip.cluster
import jip.configuration
import jip.templates

from jip.logger import getLogger, log_level
from jip.vendor.docopt import docopt

log = getLogger('jip.cli.jip_main')


def main():
    try:
        jip.configuration.install_path = os.path.abspath(
            os.path.dirname(sys.argv[0]))
    except:
        pass

    try:
        _main()
    except jip.options.ParserException as err:
        log.debug("parser error: %s", str(err), exc_info=True)
        sys.stderr.write(str(err))
        sys.exit(1)
Exemple #30
0
Options:
    -d, --db <db>  the database source that will be used to find the job
    <id>           the job id of the job that will be executed

Other Options:
    -h --help             Show this help message
"""

from jip.logger import getLogger
import jip.jobs
import jip.db
from . import parse_args
import sys
import os

log = getLogger("jip.cli.jip_exec")


def main():
    log.debug("job execution python path: %s", sys.path)
    args = parse_args(__doc__, options_first=True)
    try:
        log.info("Starting job with id %s stored in %s",
                 args['<id>'],
                 args['--db'])
        jip.db.init(path=args['--db'])
        job = jip.db.get(args['<id>'])
        if not job:
            log.error("Requested job with id %s not found!", args['<id>'])
            sys.exit(1)
        if job.state != jip.db.STATE_QUEUED:
Exemple #31
0
Issue Tracker: https://github.com/thasso/pyjip/issues
"""
import os
import sys
import jip
import jip.options
import jip.tools
import jip.cli
import jip.cluster
import jip.configuration
import jip.templates

from jip.logger import getLogger, log_level
from jip.vendor.docopt import docopt

log = getLogger('jip.cli.jip_main')


def main():
    try:
        jip.configuration.install_path = os.path.abspath(
            os.path.dirname(sys.argv[0])
        )
    except:
        pass

    try:
        _main()
    except jip.options.ParserException as err:
        log.debug("parser error: %s", str(err), exc_info=True)
        sys.stderr.write(str(err))
Exemple #32
0
    -h --help             Show this help message
"""
import os
import sys
import json

from . import parse_args
import jip
import jip.jobs
import jip.tools
import jip.options
import jip.profiles
import jip.pipelines
from jip.logger import getLogger

log = getLogger('jip.cli.jip_specs')


##################################################
# This is a bit of a hack :)
# We want the output to be printed in a specific
# order and have this working properly with 2.6 as
# well (no OrderedDict in 2.6). Therefore we subclass
# dictionary and ensure the items are yield in the order
#
# NOTE that you have to add keys manually here!!
##################################################
class sorted_dict(dict):
    def items(self):
        ks = self.keys()
        sorted_keys = [
Exemple #33
0
    -c, --cmd <cmd>          The bash command line that will be wrapped
    -h --help                Show this help message

"""
from __future__ import print_function

import jip
import jip.jobs
import jip.cluster
import jip.cli
import jip.profiles
from jip.logger import getLogger
from . import parse_args, colorize, YELLOW, RED
import sys

log = getLogger("jip.cli.jip_bash")


def main():
    args = parse_args(__doc__, options_first=False)
    pipeline = jip.Pipeline()
    bash = pipeline.job(args['--name'] if args['--name'] else 'bash').run(
        'bash')
    if not args['--cmd']:
        args['--cmd'] = "\n".join(sys.stdin.readlines())

    bash.input = [sys.stdin if a == 'stdin' else a for a in args['--input']]
    bash.output = [
        sys.stdout if a == 'stdout' else a for a in args['--output']
    ]
    bash.outfile = [a for a in args['--outfile']]
Exemple #34
0
  <args>                   optional script argument

Other Options:
    -h --help             Show this help message

"""
import sys

import jip
import jip.profiles
from . import parse_args, show_dry, show_commands, colorize, RED, \
    YELLOW
import jip.jobs
from jip.logger import getLogger

log = getLogger("jip.cli.jip_submit")


def main(argv=None):
    args = parse_args(__doc__, argv=argv)
    script_file = args["<tool>"]
    script_args = args["<args>"]
    try:
        script = jip.find(script_file)
    except LookupError, e:
        print >>sys.stderr, str(e)
        sys.exit(1)

    # load profile
    profile = jip.profiles.get(name='default'
                               if not args['--profile']
Exemple #35
0
thought the :py:func:`render_template` function.
"""
import os
from jinja2 import Environment, Undefined, contextfilter
from jip.logger import getLogger
from jip.options import Option

# contains global variable that
# will be added to any render context
# if they do not exists in the local context
global_context = None

#: the jinja2 environment
environment = None

log = getLogger('jip.templates')


def set_global_context(global_ctx):
    global global_context
    if not global_ctx:
        raise
    global_context = global_ctx


class JipUndefined(Undefined):
    """Custom undefined implementation that does not modify
    unknown variables
    """
    def __unicode__(self):
        log.info("Unknown context variable: %s", self._undefined_name)
Exemple #36
0
                             [Default: 10]
    --head                   Show the head instead of the tail
    -h --help                Show this help message
"""

from . import colorize, GREEN, RED, BLUE, STATE_COLORS
import jip.cluster
from . import parse_args, parse_job_ids

from subprocess import Popen
from os.path import exists

import jip.db
from jip.logger import getLogger

log = getLogger("jip.cli.jip_logs")


def main():
    args = parse_args(__doc__, options_first=False)
    job_ids, cluster_ids = parse_job_ids(args)
    if not job_ids and not cluster_ids:
        print(__doc__.strip("\n"))
        return
    jobs = jip.db.query(job_ids=job_ids, cluster_ids=cluster_ids,
                        archived=None, fields=['stdout', 'stderr',
                                               'job_id', 'id', 'name',
                                               'state'])
    for job in jobs:
        cluster = jip.cluster.get()
        show_log(job, cluster, args)