Esempio n. 1
0
File: proc.py Progetto: jul/dsat
def proc(cnx, arg):
    res= dict()
    sum_file = dict()
    procc=dict()
    sum_connection = dict() 
    percent_mem= dict()
    all_proc = psutil.get_process_list()
    interesting = { "ping.py", 
        "proc.py", "master.py", "tracker.py","cpu.py", "rrd.py", "csvw.py"}
    for x in all_proc:
        key = ( set(x.cmdline()) & interesting) and "me" or "other"
        try:
            procc += dict({key : 1 })
        except:
            pass


        try:
            sum_file += dict({key :x.get_num_fds()})
        except:
            pass

        try:
            sum_connection += dict({ key: sum(x.get_num_ctx_switches())})
        except:
            pass
        try:
            percent_mem += dict({key : x.get_memory_percent() })
        except:
            pass
    ratio = lambda d : min(1,1.0 *d.get("me", 0)/max(.0001,d.get("other",0)))
    absol = lambda d : d.get("me", 0) + d.get("other", 0)
    res=dict(sum_file= sum_file, percent_mem= percent_mem, all_proc =len(all_proc))
    res["data"] = map(ratio, [ sum_file, sum_connection, percent_mem, procc])
    res["data"] += map(absol, [sum_file, sum_connection, percent_mem, procc])
    return res
Esempio n. 2
0
File: proc.py Progetto: jul/dsat
def proc(ctx, payload, msg):
    res= dict()

    sum_file = dict()
    procc= dict()
    sum_connection = dict() 
    percent_mem= dict()
    all_proc = psutil.get_process_list()
    carbon_measure = dict()
    interesting = { 
        '/usr/lib/firefox/firefox', '/opt/google/chrome/chrome', 'mysqld', 
        'mongod', "ping.py", "clock.py", "orchester.py", 
        "proc.py", "master.py", "tracker.py","cpu.py", "rrd.py", "csvw.py"}
    for x in all_proc:
        try:
            key = ( set(x.cmdline()) & interesting) and "me" or "other"
            carbon_key=None
            cmd = x.cmdline()
            intersect =  interesting &set(cmd)
            if intersect:
                assert len(intersect) == 1
                carbon_key = intersect.pop()
                
            try:
                procc += dict({key : 1 })
            except:
                pass


            try:
                sum_file += dict({key :x.get_num_fds()})
            except:
                pass

            try:
                sum_connection += dict({ key: sum(x.get_num_ctx_switches())})
            except:
                pass
            try:
                percent_mem += dict({key : x.get_memory_percent() })
                if carbon_key:
                    carbon_measure += { carbon_key: x.get_memory_percent() }
            except:
                pass
        except NoSuchProcess:
            pass 
    try:
        send(carbon_measure)
    except Exception as e:
        ctx.log.error('Carbon is not very liking that %r' % e)
    ratio = lambda d : min(1,1.0 *d.get("me", 0)/max(.0001,d.get("other",0)))
    absol = lambda d : d.get("me", 0) + d.get("other", 0)
    res=dict(sum_file= sum_file, percent_mem= percent_mem, all_proc =len(all_proc))
    res["data"] = map(ratio, [ sum_file, sum_connection, percent_mem, procc])
    res["data"] += map(absol, [sum_file, sum_connection, percent_mem, procc])
    del all_proc
    return res
Esempio n. 3
0
File: tracker.py Progetto: jul/dsat
def event_listener(CNX, config):
    """Processlet responsible for routing and reacting on status change"""
    D("event listener")
    global error_per_stage, busy_per_stage, message_per_stage
    cnx = CNX


    propagated_to = dict()

    def on_failure(new):
        log.critical("KO for <%(job_id)r-%(task_id)r> %(arg)r" % new)

    def on_end( new):
        #D("JOB PROCESSED @ STEP %(step)s" % new)
        cleanUp(new)

    def cleanUp(new):
        #q_madd(timer_q,"cancel", new)
        global busy_per_stage
        busy_per_stage -= dict({ new["step"] : 1 })

    def on_begin(new):
        global busy_per_stage
        busy_per_stage += { new["step"] : 1 }


    def on_timeout(new):
        __on_error(new)

    def on_error(new):
        __on_error(new)

    def __on_error(new):
        global error_per_stage
        cleanUp(new)
        error_per_stage+={ new["step"] : 1 }
        D("OE %s" % ( _f(new) ))
        #### WHY did I put that? 
        if int(new.get("retry",0)) >= config["max_retry"]:
            D('retry %(retry)s for <%(job_id)s-%(task_id)s>' % new)
            new["event"] = "FAILURE"
            on_failure(new)
        else:
            log.critical("unhandled failure for %r" % new)
            ### could also restart failing processes here
    def on_send(new):
        pass
    
    def on_init(new):
        pass


    def on_propagate( new):
        D("just passing %s" % _f(new))
        on_propagate += { new["next"] : 1 }

    D("waiting")
    action = dict(
            INIT = False,
            BOUNCE = False,
            SEND = False,
            HAPPY_END = False,
            ACK = False,
            END = on_end,
            BEGIN = on_begin,
            PROPAGATE = on_propagate,
            ERROR = on_error,
            TIMEOUT = on_error,
            O_TIMEOUT = on_failure,
    )
    
    print("Waiting for socket to be read cf 100% CPU zmq bug")
    sleep(1)
    local_in_sox = cnx["tracker_in"]
    print LOCAL_INFO
    CLOCK_M = int(config.get("tracker_clock_every", 1000))
    ignore = 0
    check_delay = config.get("check_delay", 3) * 2
    now = time()
    while True:
        new = parse_event(local_in_sox)
        ignore += 1
        ignore %= CLOCK_M
        if not ignore:
            log.info("RCV %s" % _f(new))
            if abs(time() - now) > check_delay:
                q_madd( watchdog_q, busy_per_stage)
                log.info("zombie count %r" % busy_per_stage)
                log.info("MPS %r" % message_per_stage)
                log.info("EPS %r" % error_per_stage)
                now = time()
                
         
        if new["where"] != LOCAL_INFO["where"]:
           D("NOT FOR ME %s" % _f(new))
           continue
        try:

            message_per_stage += { new["step"] : 1 } 
        # only one message at a time can be treated not even sure I need it
            task_id = new["task_id"]
            job_id = new["job_id"]

            if int(new["seq"]) > config.get("max_seq", 50):
                logging.warning("<%r> was bounced <%r> times" %(new,
                    new["seq"]))
                continue
            # Wut ? event, U sure? 
            if action[new["event"]]:
                action[new["event"]](new)
        except Exception as e:
            D("MON new is %r" % new)
            log.exception("MON %s" % e)
Esempio n. 4
0
File: tracker.py Progetto: jul/dsat
 def cleanUp(new):
     #q_madd(timer_q,"cancel", new)
     global busy_per_stage
     busy_per_stage -= dict({ new["step"] : 1 })
Esempio n. 5
0
File: tracker.py Progetto: jul/dsat
def q_madd(q, what):
    q.put(dumps(what))

def q_mget(q):
    return loads(q.get())



D("Started tracker")

CNX = get_connection(CONFIG, LOCAL_INFO)
from pprint import PrettyPrinter as PP
pp = PP(indent=4)
P = pp.pprint
P(dict(CNX))

watchdog_q = Queue()


def watcher(watchdog_q, config):
    proc_tracker = ProcTracker(config)
    delay = config.get("check_delay",3)
    now = time()
    while True: 
        busy_per_stage = q_mget(watchdog_q)
        if abs(time() - now ) > delay:
            proc_tracker.watch(busy_per_stage)
            now = time()
    
ProcWather = Process(target=watcher, args=(watchdog_q, CONFIG,))
Esempio n. 6
0
File: tracker.py Progetto: jul/dsat
def watcher(watchdog_q, config):
    proc_tracker = ProcTracker(config)
    delay = config.get("check_delay",.1)
    now = time()
    while True: 
        busy_per_stage = q_mget(watchdog_q)
        if abs(time() - now ) > delay:
            proc_tracker.watch(busy_per_stage)
            now = time()
    
ProcWather = Process(target=watcher, args=(watchdog_q, CONFIG,))
ProcWather.start()


busy_per_stage = dict()
message_per_stage = dict()
error_per_stage = dict()
def event_listener(CNX, config):
    """Processlet responsible for routing and reacting on status change"""
    D("event listener")
    global error_per_stage, busy_per_stage, message_per_stage
    cnx = CNX


    propagated_to = dict()

    def on_failure(new):
        log.critical("KO for <%(emitter)r-%(task_id)r> %(arg)r" % new)

    def on_end( new):