Example #1
0
File: job.py Project: wehu/pydv
 def agent_checker(cls):
     for agent_id in cls.agent_status:
         if (time() - cls.agent_status[agent_id]) > cls.agent_time_out:
             if agent_id in cls.running_cmds:
                 logger.warning('agent {} time out and terminated'.format(agent_id))
                 GCFEngine.kill_agent(agent_id)
                 cmd_spec = cls.running_cmds[agent_id]
                 v = cls.agent_visitor[agent_id]
                 del cls.visitor_agent[v]
                 del cls.agent_visitor[agent_id]
                 del cls.agent_status[agent_id]
                 del cls.running_cmds[agent_id]
                 cls.push_cmd(v, cmd_spec)
     cls.timer = Timer(cls.time_out, cls.agent_checker, args=(cls,))
     cls.timer.start()
Example #2
0
 def agent_checker(cls):
     for agent_id in cls.agent_status:
         if (time() - cls.agent_status[agent_id]) > cls.agent_time_out:
             if agent_id in cls.running_cmds:
                 logger.warning(
                     'agent {} time out and terminated'.format(agent_id))
                 GCFEngine.kill_agent(agent_id)
                 cmd_spec = cls.running_cmds[agent_id]
                 v = cls.agent_visitor[agent_id]
                 del cls.visitor_agent[v]
                 del cls.agent_visitor[agent_id]
                 del cls.agent_status[agent_id]
                 del cls.running_cmds[agent_id]
                 cls.push_cmd(v, cmd_spec)
     cls.timer = Timer(cls.time_out, cls.agent_checker, args=(cls, ))
     cls.timer.start()
Example #3
0
 def get_agent(cls, v):
     if v in cls.visitor_agent:
         return cls.visitor_agent[v]
     else:
         if len(cls.idle_agents) > 0:
             agent_id = cls.idle_agents.pop()
             cls.reused_agents.add(agent_id)
             vv = cls.agent_visitor[agent_id]
             del cls.visitor_agent[vv]
             del cls.agent_visitor[agent_id]
             del cls.agent_status[agent_id]
         else:
             cls.agent_count += 1
             cls.agent_md5.update(bytes(cls.agent_count))
             agent_id = cls.agent_md5.hexdigest()
             GCFEngine.spawn_agent(agent_id)
         cls.visitor_agent[v] = agent_id
         cls.agent_visitor[agent_id] = v
         cls.agent_status[agent_id] = time()
         return agent_id
Example #4
0
File: job.py Project: wehu/pydv
 def get_agent(cls, v):
     if v in cls.visitor_agent:
         return cls.visitor_agent[v]
     else:
         if len(cls.idle_agents) > 0:
             agent_id = cls.idle_agents.pop()
             cls.reused_agents.add(agent_id)
             vv = cls.agent_visitor[agent_id]
             del cls.visitor_agent[vv]
             del cls.agent_visitor[agent_id]
             del cls.agent_status[agent_id]
         else:
             cls.agent_count += 1
             cls.agent_md5.update(bytes(cls.agent_count))
             agent_id = cls.agent_md5.hexdigest()
             GCFEngine.spawn_agent(agent_id)
         cls.visitor_agent[v] = agent_id
         cls.agent_visitor[agent_id] = v
         cls.agent_status[agent_id] = time()
         return agent_id
Example #5
0
    def require_cmd(cls, data):
        agent_id = data['agent_id']
        agent_host = data['agent_host']
        agent_port = data['agent_port']
        cmd_spec = {'cmd': 'terminate'}
        if agent_id in cls.pending_cmds:
            cmd_spec = cls.pending_cmds[agent_id]
            cmd_spec['logfile'] = path.join(cls.out_dir, 'cmds', agent_id)
            del cls.pending_cmds[agent_id]
            if agent_id in cls.reused_agents:
                cls.reused_agents.remove(agent_id)
            cls.running_cmds[agent_id] = cmd_spec

        out_data = marshal.dumps(cmd_spec)
        re_try = 10
        while True:
            try:
                s = socket.socket()
                s.connect((agent_host, agent_port))
                try:
                    s.send(out_data)
                finally:
                    s.close()
                break
            except Exception as e:
                re_try -= 1
                logger.warning(str(e))
                if re_try > 0:
                    pass
                else:
                    raise e

        if agent_id not in cls.running_cmds and agent_id not in cls.reused_agents:
            if agent_id in cls.idle_agents:
                cls.idle_agents.remove(agent_id)
            if agent_id in cls.agent_visitor:
                GCFEngine.kill_agent(agent_id)
                v = cls.agent_visitor[agent_id]
                del cls.visitor_agent[v]
                del cls.agent_visitor[agent_id]
                del cls.agent_status[agent_id]
Example #6
0
File: job.py Project: wehu/pydv
    def require_cmd(cls, data):
        agent_id = data['agent_id']
        agent_host = data['agent_host']
        agent_port = data['agent_port']
        cmd_spec = {'cmd' : 'terminate'}
        if agent_id in cls.pending_cmds:
            cmd_spec = cls.pending_cmds[agent_id]
            cmd_spec['logfile'] = path.join(cls.out_dir, 'cmds', agent_id)
            del cls.pending_cmds[agent_id]
            if agent_id in cls.reused_agents:
                cls.reused_agents.remove(agent_id)
            cls.running_cmds[agent_id] = cmd_spec
       
        out_data = marshal.dumps(cmd_spec)
        re_try = 10
        while True:
            try:
                s = socket.socket()
                s.connect((agent_host, agent_port))
                try:
                    s.send(out_data)
                finally:
                    s.close()
                break
            except Exception as e:
                re_try -= 1
                logger.warning(str(e))
                if re_try > 0:
                    pass
                else:
                    raise e

        if agent_id not in cls.running_cmds and agent_id not in cls.reused_agents:
            if agent_id in cls.idle_agents:
                cls.idle_agents.remove(agent_id)
            if agent_id in cls.agent_visitor:
                GCFEngine.kill_agent(agent_id)
                v = cls.agent_visitor[agent_id]
                del cls.visitor_agent[v]
                del cls.agent_visitor[agent_id]
                del cls.agent_status[agent_id]
Example #7
0
 def cleanup(cls):
     if cls.timer:
         cls.timer.cancel()
     for agent_id in cls.agent_visitor:
         GCFEngine.kill_agent(agent_id)
Example #8
0
File: dv.py Project: wehu/pydv
def main():

    global server_p

    # parsing arguments
    (opts, args) = args_parse()

    in_q = Queue()
    out_q = Queue()

    logger.info('running dv.py')
    # start agent server
    #loop = asyncio.get_event_loop()
    server_p = Process(target=start_agent_server,
                       args=(
                           in_q,
                           out_q,
                           path.abspath(opts.out_dir),
                           opts.verbose,
                       ))
    #server_p = Thread(target=start_agent_server, args=(loop, in_q, out_q,))
    server_p.start()

    try:
        # waiting for server started
        host, port = in_q.get()

        #logger.info("agent server started on {}:{}".format(host, port))

        # set gcf engine
        if opts.gcf == 'local':
            GCFEngine.set_imp(
                Local(host, port, path.abspath(opts.out_dir), opts.verbose))
        else:
            if opts.gcf == 'lsf':
                GCFEngine.set_imp(
                    LSF(host, port, path.abspath(opts.out_dir), opts.verbose))
            else:
                raise Exception('unsupported gcf engine {}'.format(opts.gcf))

        # config job engine
        JobEngine.connect(in_q, out_q)
        JobEngine.out_dir = path.abspath(opts.out_dir)
        logger.info('max agents = {}'.format(opts.max_agents))
        JobEngine.max_cmds = opts.max_agents

        # load files
        require('loader')
        if opts.patchfile:
            for f in opts.patchfile:
                require(f)

        # evaluate experssions
        @visitor
        def top():
            @join
            def body(self):
                if opts.expr:
                    for e in opts.expr:

                        @spawn(self)
                        def body(ee=e):
                            res = eval(ee, get_ns(), get_ns())
                            if type(res) == GeneratorType:
                                yield from res
                            return res

                if opts.test:

                    @spawn(self)
                    def body():
                        res = run_test(*opts.test,
                                       action=opts.action,
                                       where=opts.where)
                        if type(res) == GeneratorType:
                            yield from res
                        return res

            yield from body()

        # run
        while True:
            JobEngine.run()
            Scheduler.run()
            if JobEngine.is_waiting() or Scheduler.is_waiting():
                next
            else:
                break
        for t in Test.test_status:
            if Test.test_status[t] == 'passed':
                logger.info("*** test '{}' passed".format(t))
            else:
                logger.error("*** test '{}' failed".format(t))
        if top.exception:

            def print_exception(e, indent=0):
                if isinstance(e, Exception):
                    for l in extract_tb(e.__traceback__):
                        logger.debug((" " * indent) + str(l))
                if not isinstance(e, Exception):
                    logger.error((" " * indent) + str(e))
                    return
                for i in e.args:
                    if not isinstance(i, list):
                        i = [i]
                    for j in i:
                        print_exception(j, indent + 2)

            print_exception(top.exception)
            logger.error('dv.py failed')
            #raise top.exception
        else:
            logger.info('dv.py passed')
    finally:
        event.notify('dvpy_done')
        cleanup()
Example #9
0
File: job.py Project: wehu/pydv
 def cleanup(cls):
     if cls.timer:
         cls.timer.cancel()
     for agent_id in cls.agent_visitor:
         GCFEngine.kill_agent(agent_id)
Example #10
0
File: dv.py Project: wehu/pydv
def main():

    global server_p

    # parsing arguments
    (opts, args) = args_parse()

    in_q  = Queue()
    out_q = Queue()

    logger.info('running dv.py')
    # start agent server
    #loop = asyncio.get_event_loop()
    server_p = Process(target=start_agent_server, args=(in_q, out_q, path.abspath(opts.out_dir), opts.verbose,))
    #server_p = Thread(target=start_agent_server, args=(loop, in_q, out_q,))
    server_p.start()

    try:
        # waiting for server started
        host, port = in_q.get()

        #logger.info("agent server started on {}:{}".format(host, port))

        # set gcf engine    
        if opts.gcf == 'local':
            GCFEngine.set_imp(Local(host, port, path.abspath(opts.out_dir), opts.verbose))
        else:
            if opts.gcf == 'lsf':
                GCFEngine.set_imp(LSF(host, port, path.abspath(opts.out_dir), opts.verbose))
            else:
                raise Exception('unsupported gcf engine {}'.format(opts.gcf))
    
        # config job engine
        JobEngine.connect(in_q, out_q)
        JobEngine.out_dir = path.abspath(opts.out_dir)
        logger.info('max agents = {}'.format(opts.max_agents))
        JobEngine.max_cmds = opts.max_agents

        # load files
        require('loader')
        if opts.patchfile:
            for f in opts.patchfile:
                require(f)

        # evaluate experssions
        @visitor
        def top():
            @join
            def body(self):
                if opts.expr:
                    for e in opts.expr:
                        @spawn(self)
                        def body(ee=e):
                            res = eval(ee, get_ns(), get_ns())
                            if type(res) == GeneratorType:
                                yield from res
                            return res
                if opts.test:
                    @spawn(self)
                    def body():
                        res = run_test(*opts.test, action=opts.action, where=opts.where)
                        if type(res) == GeneratorType:
                            yield from res
                        return res
            yield from body()

        # run
        while True:
            JobEngine.run()
            Scheduler.run()
            if JobEngine.is_waiting() or Scheduler.is_waiting():
                next
            else:
                break
        for t in Test.test_status:
            if Test.test_status[t] == 'passed':
                logger.info("*** test '{}' passed".format(t))
            else:
                logger.error("*** test '{}' failed".format(t))
        if top.exception:
            def print_exception(e, indent=0):
                if isinstance(e, Exception):
                    for l in extract_tb(e.__traceback__):
                        logger.debug((" "*indent)+str(l))
                if not isinstance(e, Exception):
                    logger.error((" "*indent)+str(e))
                    return
                for i in e.args:
                    if not isinstance(i, list):
                        i = [i]
                    for j in i:
                        print_exception(j, indent+2)
            print_exception(top.exception)
            logger.error('dv.py failed')
            #raise top.exception
        else:
            logger.info('dv.py passed')
    finally:
        event.notify('dvpy_done')
        cleanup()