Exemple #1
0
def debug_exception():
    '''
    Start a debugger on the exception currently being handled
    '''
    import sys

    _,_, tb = sys.exc_info()
    debugger = _get_debugger()
    debugger.reset()
    debugger.interaction(None, tb)
Exemple #2
0
def post_mortem(tb=None):
    ''' Helper function, like pdb.post_mortem '''
    # handling the default
    if tb is None:
        # sys.exc_info() returns (type, value, traceback) if an exception is
        # being handled, otherwise it returns None
        tb = sys.exc_info()[2]
        if tb is None:
            raise ValueError("A valid traceback must be passed if no "
                             "exception is being handled")

    debugger = Vdb()
    debugger.reset()
    debugger.interaction(None, tb)
Exemple #3
0
def execution_loop(tasks, options):
    from time import sleep

    logging.info('Execute start (%s tasks)' % len(tasks))

    # For the special (but common) case where most (if not all) of the tasks
    # can be loaded directly, just skip them as fast as possible:
    first_unloadable = 0
    while (first_unloadable <
           len(tasks)) and tasks[first_unloadable].can_load():
        t = tasks[first_unloadable]
        jug_hook('execute.task-loadable', (tasks[first_unloadable], ))
        first_unloadable += 1
    del tasks[:first_unloadable]

    prevtask = None
    while tasks:
        upnext = []  # tasks that can be run
        nr_wait_cycles = int(options.execute_nr_wait_cycles)
        for i in range(nr_wait_cycles):
            max_cannot_run = min(len(tasks), 128)
            if i == nr_wait_cycles - 1:
                # in the last loop iteration, check all tasks to ensure we don't miss any
                max_cannot_run = len(tasks)
            for i in range(max_cannot_run):
                # The argument for this is the following:
                # if T' is dependent on the result of T, it is better if the
                # processor that ran T, also runs T'. By having everyone else
                # push T' to the end of tasks, this is more likely to happen.
                #
                # Furthermore, this avoids always querying the same tasks.
                if tasks[0].can_run():
                    break
                tasks.append(tasks.pop(0))
            while tasks and tasks[0].can_run():
                upnext.append(tasks.pop(0))
            if upnext:
                break
            for ti, t in enumerate(tasks):
                if t.can_run():
                    upnext.append(tasks.pop(ti))
                    break
            if upnext:
                break
            logging.info('waiting %s secs for an open task...' %
                         options.execute_wait_cycle_time_secs)
            sleep(int(options.execute_wait_cycle_time_secs))
        if not upnext:
            logging.info('No tasks can be run!')
            break
        for t in upnext:
            if t.can_load():
                jug_hook('execute.task-loadable', (t, ))
                continue
            locked = False
            try:
                locked = t.lock()
                if t.can_load(
                ):  # This can be true if the task ran between the check above and this one
                    jug_hook('execute.task-loadable', (t, ))
                elif locked:
                    logging.info('Executing %s...' % t.name)
                    jug_hook('execute.task-pre-execute', (t, ))

                    if options.aggressive_unload:
                        if prevtask is not None:
                            active = set([id(d) for d in t.dependencies()])
                            for d in itertools.chain(prevtask.dependencies(),
                                                     [prevtask]):
                                if id(d) not in active:
                                    d.unload()
                        prevtask = t
                    t.run(debug_mode=options.debug)
                    jug_hook('execute.task-executed1', (t, ))
                else:
                    logging.info('Already in execution %s...' % t.name)
            except SystemExit:
                raise
            except Exception as e:
                if options.pdb:
                    import sys
                    _, _, tb = sys.exc_info()

                    # The code below is a complex attempt to load IPython
                    # debugger which works with multiple versions of IPython.
                    #
                    # Unfortunately, their API kept changing prior to the 1.0.
                    try:
                        import IPython
                        try:
                            import IPython.core.debugger
                            try:
                                from IPython.terminal.ipapp import load_default_config
                                config = load_default_config()
                                colors = config.TerminalInteractiveShell.colors
                            except:
                                import IPython.core.ipapi
                                ip = IPython.core.ipapi.get()
                                colors = ip.colors
                            try:
                                debugger = IPython.core.debugger.Pdb(
                                    colors.get_value(initial='Linux'))
                            except AttributeError:
                                debugger = IPython.core.debugger.Pdb(colors)
                        except ImportError:
                            #Fallback to older version of IPython API
                            import IPython.ipapi
                            import IPython.Debugger
                            shell = IPython.Shell.IPShell(argv=[''])
                            ip = IPython.ipapi.get()
                            debugger = IPython.Debugger.Pdb(ip.options.colors)
                    except ImportError:
                        #Fallback to standard debugger
                        import pdb
                        debugger = pdb.Pdb()

                    debugger.reset()
                    debugger.interaction(None, tb)
                else:
                    logging.critical('Exception while running %s: %s' %
                                     (t.name, e))
                    for other in itertools.chain(upnext, tasks):
                        for dep in other.dependencies():
                            if dep is t:
                                logging.critical(
                                    'Other tasks are dependent on this one! Parallel processors will be held waiting!'
                                )
                if not options.execute_keep_going:
                    raise
            finally:
                if locked: t.unlock()
            if options.aggressive_unload and prevtask is not None:
                prevtask.unload()
Exemple #4
0
def execution_loop(tasks, options, tasks_executed, tasks_loaded):
    from time import sleep

    logging.info('Execute start (%s tasks)' % len(tasks))
    while tasks:
        upnext = [] # tasks that can be run
        for i in range(int(options.execute_nr_wait_cycles)):
            max_cannot_run = min(len(tasks), 128)
            for i in range(max_cannot_run):
                # The argument for this is the following:
                # if T' is dependent on the result of T, it is better if the
                # processor that ran T, also runs T'. By having everyone else
                # push T' to the end of tasks, this is more likely to happen.
                #
                # Furthermore, this avoids always querying the same tasks.
                if tasks[0].can_run():
                    break
                tasks.append(tasks.pop(0))
            while tasks and tasks[0].can_run():
                upnext.append(tasks.pop(0))
            if upnext:
                break
            for ti,t in enumerate(tasks):
                if t.can_run():
                    upnext.append(tasks.pop(ti))
                    break
            if upnext:
                break
            logging.info('waiting %s secs for an open task...' % options.execute_wait_cycle_time_secs)
            sleep(int(options.execute_wait_cycle_time_secs))
        if not upnext:
            logging.info('No tasks can be run!')
            break
        for t in upnext:
            if t.can_load():
                logging.info('Loadable %s...' % t.name)
                tasks_loaded[t.name] += 1
                continue
            locked = False
            try:
                locked = t.lock()
                if t.can_load(): # This can be true if the task ran between the check above and this one
                    logging.info('Loadable %s...' % t.name)
                    tasks_loaded[t.name] += 1
                elif locked:
                    logging.info('Executing %s...' % t.name)
                    t.run(debug_mode=options.debug)
                    tasks_executed[t.name] += 1
                    if options.aggressive_unload:
                        t.unload_recursive()
                else:
                    logging.info('Already in execution %s...' % t.name)
            except Exception as e:
                if options.pdb:
                    import sys
                    _,_, tb = sys.exc_info()

                    # The code below is a complex attempt to load IPython
                    # debugger which works with multiple versions of IPython.
                    #
                    # Unfortunately, their API kept changing prior to the 1.0.
                    try:
                        import IPython
                        try:
                            import IPython.core.debugger
                            try:
                                from IPython.terminal.ipapp import load_default_config
                                config = load_default_config()
                                colors = config.TerminalInteractiveShell.colors
                            except:
                                import IPython.core.ipapi
                                ip = IPython.core.ipapi.get()
                                colors = ip.colors
                            debugger = IPython.core.debugger.Pdb(colors)
                        except ImportError:
                            #Fallback to older version of IPython API
                            import IPython.ipapi
                            import IPython.Debugger
                            shell = IPython.Shell.IPShell(argv=[''])
                            ip = IPython.ipapi.get()
                            debugger=IPythong.Debugger.Pdb(ip.options.colors)
                    except ImportError:
                        #Fallback to standard debugger
                        import pdb
                        debugger = pdb.Pdb()

                    debugger.reset()
                    debugger.interaction(None, tb)
                else:
                    import itertools
                    logging.critical('Exception while running %s: %s' % (t.name,e))
                    for other in itertools.chain(upnext, tasks):
                        for dep in other.dependencies():
                            if dep is t:
                                logging.critical('Other tasks are dependent on this one! Parallel processors will be held waiting!')
                if not options.execute_keep_going:
                    raise
            finally:
                if locked: t.unlock()
Exemple #5
0
def execution_loop(tasks, options):
    from time import sleep

    logging.info('Execute start (%s tasks)' % len(tasks))
    while tasks:
        upnext = []  # tasks that can be run
        for i in range(int(options.execute_nr_wait_cycles)):
            max_cannot_run = min(len(tasks), 128)
            for i in range(max_cannot_run):
                # The argument for this is the following:
                # if T' is dependent on the result of T, it is better if the
                # processor that ran T, also runs T'. By having everyone else
                # push T' to the end of tasks, this is more likely to happen.
                #
                # Furthermore, this avoids always querying the same tasks.
                if tasks[0].can_run():
                    break
                tasks.append(tasks.pop(0))
            while tasks and tasks[0].can_run():
                upnext.append(tasks.pop(0))
            if upnext:
                break
            for ti, t in enumerate(tasks):
                if t.can_run():
                    upnext.append(tasks.pop(ti))
                    break
            if upnext:
                break
            logging.info('waiting %s secs for an open task...' %
                         options.execute_wait_cycle_time_secs)
            sleep(int(options.execute_wait_cycle_time_secs))
        if not upnext:
            logging.info('No tasks can be run!')
            break
        for t in upnext:
            if t.can_load():
                logging.info('Loadable %s...' % t.name)
                jug_hook('execute.task-loadable', (t, ))
                continue
            locked = False
            try:
                locked = t.lock()
                if t.can_load(
                ):  # This can be true if the task ran between the check above and this one
                    logging.info('Loadable %s...' % t.name)
                    jug_hook('execute.task-loadable', (t, ))
                elif locked:
                    logging.info('Executing %s...' % t.name)
                    jug_hook('execute.task-pre-execute', (t, ))

                    t.run(debug_mode=options.debug)
                    jug_hook('execute.task-executed1', (t, ))
                else:
                    logging.info('Already in execution %s...' % t.name)
            except SystemExit:
                raise
            except Exception as e:
                if options.pdb:
                    import sys
                    _, _, tb = sys.exc_info()

                    # The code below is a complex attempt to load IPython
                    # debugger which works with multiple versions of IPython.
                    #
                    # Unfortunately, their API kept changing prior to the 1.0.
                    try:
                        import IPython
                        try:
                            import IPython.core.debugger
                            try:
                                from IPython.terminal.ipapp import load_default_config
                                config = load_default_config()
                                colors = config.TerminalInteractiveShell.colors
                            except:
                                import IPython.core.ipapi
                                ip = IPython.core.ipapi.get()
                                colors = ip.colors
                            debugger = IPython.core.debugger.Pdb(colors)
                        except ImportError:
                            #Fallback to older version of IPython API
                            import IPython.ipapi
                            import IPython.Debugger
                            shell = IPython.Shell.IPShell(argv=[''])
                            ip = IPython.ipapi.get()
                            debugger = IPython.Debugger.Pdb(ip.options.colors)
                    except ImportError:
                        #Fallback to standard debugger
                        import pdb
                        debugger = pdb.Pdb()

                    debugger.reset()
                    debugger.interaction(None, tb)
                else:
                    import itertools
                    logging.critical('Exception while running %s: %s' %
                                     (t.name, e))
                    for other in itertools.chain(upnext, tasks):
                        for dep in other.dependencies():
                            if dep is t:
                                logging.critical(
                                    'Other tasks are dependent on this one! Parallel processors will be held waiting!'
                                )
                if not options.execute_keep_going:
                    raise
            finally:
                if locked: t.unlock()
Exemple #6
0
def execution_loop(tasks, options):
    from time import sleep

    logging.info('Execute start (%s tasks)' % len(tasks))

    # For the special (but common) case where most (if not all) of the tasks
    # can be loaded directly, just skip them as fast as possible:
    first_unloadable = 0
    while (first_unloadable < len(tasks)) and tasks[first_unloadable].can_load():
        t = tasks[first_unloadable]
        jug_hook('execute.task-loadable', (tasks[first_unloadable],))
        first_unloadable += 1
    del tasks[:first_unloadable]

    while tasks:
        upnext = [] # tasks that can be run
        nr_wait_cycles = int(options.execute_nr_wait_cycles)
        for i in range(nr_wait_cycles):
            max_cannot_run = min(len(tasks), 128)
            if i == nr_wait_cycles - 1:
                # in the last loop iteration, check all tasks to ensure we don't miss any
                max_cannot_run = len(tasks)
            for i in range(max_cannot_run):
                # The argument for this is the following:
                # if T' is dependent on the result of T, it is better if the
                # processor that ran T, also runs T'. By having everyone else
                # push T' to the end of tasks, this is more likely to happen.
                #
                # Furthermore, this avoids always querying the same tasks.
                if tasks[0].can_run():
                    break
                tasks.append(tasks.pop(0))
            while tasks and tasks[0].can_run():
                upnext.append(tasks.pop(0))
            if upnext:
                break
            for ti,t in enumerate(tasks):
                if t.can_run():
                    upnext.append(tasks.pop(ti))
                    break
            if upnext:
                break
            logging.info('waiting %s secs for an open task...' % options.execute_wait_cycle_time_secs)
            sleep(int(options.execute_wait_cycle_time_secs))
        if not upnext:
            logging.info('No tasks can be run!')
            break
        for t in upnext:
            if t.can_load():
                jug_hook('execute.task-loadable', (t,))
                continue
            locked = False
            try:
                locked = t.lock()
                if t.can_load(): # This can be true if the task ran between the check above and this one
                    jug_hook('execute.task-loadable', (t,))
                elif locked:
                    logging.info('Executing %s...' % t.name)
                    jug_hook('execute.task-pre-execute', (t,))

                    t.run(debug_mode=options.debug)
                    jug_hook('execute.task-executed1', (t,))
                else:
                    logging.info('Already in execution %s...' % t.name)
            except SystemExit:
                raise
            except Exception as e:
                if options.pdb:
                    import sys
                    _,_, tb = sys.exc_info()

                    # The code below is a complex attempt to load IPython
                    # debugger which works with multiple versions of IPython.
                    #
                    # Unfortunately, their API kept changing prior to the 1.0.
                    try:
                        import IPython
                        try:
                            import IPython.core.debugger
                            try:
                                from IPython.terminal.ipapp import load_default_config
                                config = load_default_config()
                                colors = config.TerminalInteractiveShell.colors
                            except:
                                import IPython.core.ipapi
                                ip = IPython.core.ipapi.get()
                                colors = ip.colors
                            try:
                                debugger = IPython.core.debugger.Pdb(colors.get_value(initial='Linux'))
                            except AttributeError:
                                debugger = IPython.core.debugger.Pdb(colors)
                        except ImportError:
                            #Fallback to older version of IPython API
                            import IPython.ipapi
                            import IPython.Debugger
                            shell = IPython.Shell.IPShell(argv=[''])
                            ip = IPython.ipapi.get()
                            debugger = IPython.Debugger.Pdb(ip.options.colors)
                    except ImportError:
                        #Fallback to standard debugger
                        import pdb
                        debugger = pdb.Pdb()

                    debugger.reset()
                    debugger.interaction(None, tb)
                else:
                    import itertools
                    logging.critical('Exception while running %s: %s' % (t.name,e))
                    for other in itertools.chain(upnext, tasks):
                        for dep in other.dependencies():
                            if dep is t:
                                logging.critical('Other tasks are dependent on this one! Parallel processors will be held waiting!')
                if not options.execute_keep_going:
                    raise
            finally:
                if locked: t.unlock()