def load_procedures(self): if os.path.exists("AVMaster/conf/procedures.yaml"): Procedure.load_from_file("AVMaster/conf/procedures.yaml") confs = glob.glob("AVMaster/conf/procedures/*.yaml") for conf in confs: logging.info("Loading conf: %s" % conf) Procedure.load_from_file(conf) if not Procedure.check(): raise SyntaxError("Errors in procedures")
def test_dispatcher(): c = command.factory(["BEGIN", True, ['whatever', 'end']]) agentFiles = "" params = "" update = Procedure("UPDATE", ["REVERT", "START_VM", "RELOG", "STOP_VM"]) dispatch = Procedure("DISPATCH", ["REVERT", "START_VM", ("PUSH", None, agentFiles)]) scout = Procedure("SCOUT", [ ("CALL", None, "dispatch"), ("PUSH", None, agentFiles), ]) assert update assert dispatch assert scout
def test_procedure_insert(): c = command.factory(["BEGIN", True, ['whatever', 'end']]) agentFiles = "" params = "" p1 = Procedure("UPDATE", ["REVERT", "START_VM", "RELOG", "STOP_VM"]) p2 = Procedure("DISPATCH", ["REVERT", "START_VM", ("PUSH", None, agentFiles)]) lp1= len(p1) lp2= len(p2) p1.insert(p2) assert p1 assert p2 assert len(p1) == lp1 + lp2
def notest_ProtocolEval(): host = "localhost" mq = MQStar(host) mq.clean() c = "client1" mq.add_client(c) commands = [ "BEGIN", ("EVAL_SERVER", "dir()"), ("EVAL_SERVER", "locals()"), ("EVAL_SERVER", "__import__('os').getcwd()"), ("END", None, None) ] procedure = Procedure("PROC", commands) p = Protocol(mq, c, procedure) while p.send_next_command(): logging.debug("sent command") exit = False while not exit: rec = mq.receive_server(blocking=True, timeout=10) if rec is not None: logging.debug("- SERVER RECEIVED %s %s" % (rec, type(rec))) c, msg = rec answer = p.receive_answer(c, msg) logging.debug("- SERVER RECEIVED ANSWER: ", answer.success) if answer.name == "END" or not answer.success: logging.debug("- SERVER RECEIVE END") #if answer.success: a = """('client1', ('EVAL_SERVER', True, {'self': <Command_EVAL_SERVER.Command_EVAL_SERVER object at 0x10931f810>, 'args': 'locals()'}))""" # p.send_next_command() else: logging.debug("- SERVER RECEIVED empty") exit = True
def test_ProtocolCall(): host = "localhost" mq = MQStar(host) mq.clean() c = "client1" mq.add_client(c) yaml = """ HELLO: - EVAL_SERVER: 3+2 BASIC: - EVAL_SERVER: dir() - CALL: HELLO CALLER: - BEGIN - CALL: BASIC - EVAL_SERVER: locals() - END """ procedures = Procedure.load_from_yaml(yaml) caller = Procedure.procedures["CALLER"] basic = Procedure.procedures["BASIC"] class D: pass d = D() d.mq = mq p = Protocol(d, c, caller) while p.send_next_command(): logging.debug("sent command") exit = False answers =0 while not exit: rec = mq.receive_server(blocking=True, timeout=10) if rec is not None: logging.debug("- SERVER RECEIVED %s %s" % (rec, type(rec))) c, msg = rec command_unserialize = command.unserialize(msg) answer = p.receive_answer(c, command_unserialize) logging.debug("- SERVER RECEIVED ANSWER: %s" % answer.success) if answer.success: answers += 1 if answer.name == "END" or not answer.success: logging.debug("- SERVER RECEIVE END") #if answer.success: exit = True else: logging.debug("- SERVER RECEIVED empty") exit = True assert answers == 7, "wrong answers: %s" % answers
def test_avagent_pull(): host = "localhost" vms = ["testvm_%d" % i for i in range(1)] #command_client={ 'COMMAND_CLIENT': [{ 'SET': [ 'windows' 'whatever']}]} procedure = """ TEST: - START_AGENT - SET: - [backend, 192.168.100.201] - [frontend, 172.20.100.204] - [redis, 10.0.20.1] - BUILD: [ pull, windows, silent] - STOP_AGENT """ test = Procedure.load_from_yaml(procedure) host = "localhost" mq = MQStar(host) mq.clean() logging.debug("MQ session: %s" % mq.session) #istanzia n client e manda delle procedure. vm_manager.vm_conf_file = "../AVMaster/conf/vms.cfg" report = Report() # dispatcher, inoltra e riceve i comandi della procedura test sulle vm dispatcher = Dispatcher(mq, vms, report) thread = threading.Thread(target=dispatcher.dispatch, args=(test["TEST"], )) thread.start() #p = Process(target=dispatcher.dispatch, args=(test["TEST"],)) #p.start() # i client vengono eseguiti asincronicamente e comunicano tramite redis al server pool = Pool(len(vms)) r = pool.map_async(av_agent.start_agent, ((v, host, mq.session) for v in vms)) r.get( ) #notare che i results dei client non ci interessano, viaggia tutto su channel/command. # chiusura del server #p.join() thread.join() logging.debug(dispatcher.report) logging.debug("sent: %s" % dispatcher.report.c_sent) logging.debug("received: %s" % Report.c_received)
def test_report_meta(): yaml = """ P1: - SLEEP: 1 P2: - SLEEP P3: - SLEEP: 2 P4: - SLEEP TEST_REPORT: - VM: [noav,zenoav] - SET_SERVER: mail_recipients: [[email protected]] - REPORT: - P1: ["AVtest", "MyCase"] - P2: ["AVtest", "MyOtherCase", INVERSE ] - P3 - P4 """ procedures = Procedure.load_from_yaml(yaml) vms = ["noav", "zenovm"] #vms = ["noav"] redis_host = "localhost" mq = MQStar(redis_host) mq.clean() vm_manager.vm_conf_file = "../AVMaster/conf/vms.cfg" dispatcher = Dispatcher(mq, vms) logging.info("STARTING TEST REPORT") dispatcher.dispatch(procedures["TEST_REPORT"]) logging.info("STOPPING TEST REPORT") r = report.Report() #report.finish() assert r assert len(r.c_received) == len(vms) assert len(r.c_sent) == len(vms) assert len(r.current_procedure) == len(vms) assert len(r.c_received) == len(vms), len(r.c_received) for vm in vms: assert vm in r.c_received assert len(r.c_received[vm]) >= 18
def test_report_meta(): yaml = """ P1: - SLEEP: 1 P2: - SLEEP P3: - SLEEP: 2 P4: - SLEEP TEST_REPORT: - VM: [noav,zenoav] - SET_SERVER: mail_recipients: [[email protected]] - REPORT: - P1: ["AVtest", "MyCase"] - P2: ["AVtest", "MyOtherCase", INVERSE ] - P3 - P4 """ procedures = Procedure.load_from_yaml(yaml) vms = ["noav", "zenovm"] # vms = ["noav"] redis_host = "localhost" mq = MQStar(redis_host) mq.clean() vm_manager.vm_conf_file = "../AVMaster/conf/vms.cfg" dispatcher = Dispatcher(mq, vms) logging.info("STARTING TEST REPORT") dispatcher.dispatch(procedures["TEST_REPORT"]) logging.info("STOPPING TEST REPORT") r = report.Report() # report.finish() assert r assert len(r.c_received) == len(vms) assert len(r.c_sent) == len(vms) assert len(r.current_procedure) == len(vms) assert len(r.c_received) == len(vms), len(r.c_received) for vm in vms: assert vm in r.c_received assert len(r.c_received[vm]) >= 18
def test_avagent_pull(): host = "localhost" vms = [ "testvm_%d" % i for i in range(1) ] #command_client={ 'COMMAND_CLIENT': [{ 'SET': [ 'windows' 'whatever']}]} procedure = """ TEST: - START_AGENT - SET: - [backend, 192.168.100.201] - [frontend, 172.20.100.204] - [redis, 10.0.20.1] - BUILD: [ pull, windows, silent] - STOP_AGENT """ test = Procedure.load_from_yaml(procedure) host = "localhost" mq = MQStar(host) mq.clean() logging.debug("MQ session: %s" % mq.session) #istanzia n client e manda delle procedure. vm_manager.vm_conf_file = "../AVMaster/conf/vms.cfg" report= Report() # dispatcher, inoltra e riceve i comandi della procedura test sulle vm dispatcher = Dispatcher(mq, vms, report) thread = threading.Thread(target=dispatcher.dispatch, args=(test["TEST"],)) thread.start() #p = Process(target=dispatcher.dispatch, args=(test["TEST"],)) #p.start() # i client vengono eseguiti asincronicamente e comunicano tramite redis al server pool = Pool(len(vms)) r = pool.map_async(av_agent.start_agent, ( (v, host, mq.session) for v in vms) ) r.get() #notare che i results dei client non ci interessano, viaggia tutto su channel/command. # chiusura del server #p.join() thread.join() logging.debug(dispatcher.report) logging.debug("sent: %s" % dispatcher.report.c_sent) logging.debug("received: %s" % Report.c_received)
def test_dispatcher_server(): host = "localhost" vms = ["noav", "zenovm"] host = "localhost" mq = MQStar(host) mq.clean() #istanzia n client e manda delle procedure. vm_manager.vm_conf_file = "../AVMaster/conf/vms.cfg" dispatcher = Dispatcher(mq, vms) test = Procedure("TEST", [("EVAL_SERVER", None, 'vm'), ("SLEEP", None, 10)]) dispatcher.dispatch(test)
def execute(vm, protocol, mon_args): """ client side, returns (bool,*) """ logging.debug(" SET %s" % str(mon_args)) assert vm, "null vm" assert command.context is not None assert isinstance(mon_args, list), "VM expects a list" assert protocol assert protocol.procedure logging.debug("insert report init") command_list = [] command_list.append("REPORT_INIT") for proc_token in mon_args: report_args = [] if isinstance(proc_token, basestring): proc_name = proc_token elif isinstance(proc_token, dict): assert len(proc_token.keys()) == 1 proc_name = proc_token.keys()[0] report_args = proc_token[proc_name] else: return False, "Error parsing" logging.debug("insert report kind: %s args: %s" % (proc_name, report_args)) command_list.append(["REPORT_KIND_INIT", None, (proc_name)]) command_list.append(["CALL", None, (proc_name)]) command_list.append( ["REPORT_KIND_END", None, (proc_name, report_args)]) command_list.append("REPORT_END") proc = Procedure("_REPORT", command_list) protocol.procedure.insert(proc) #logging.debug("procedure: %s" % (protocol.procedure.command_list)) #logging.debug("report items: %s" % (command.context)) return True, "REPORT"
def test_procedure_yaml(): yaml = """UPDATE: - REVERT - START_VM - RELOG - STOP_VM DISPATCH: - REVERT - START_VM - RELOG - PUSH: - file.sh - anotherfile.sh SCOUT: - CALL: DISPATCH - START_AGENT - BUILD """ init_proc_number = len(Procedure.procedures) procedures = Procedure.load_from_yaml(yaml) assert procedures, "empty procedures" logging.debug("procedures: %s" % procedures) assert len(procedures) == 3, "wrong procedures number: %s" % len(procedures) for p in procedures.values(): assert isinstance(p, Procedure), "not a Procedure: %s" % p assert p.name assert p.command_list assert len(p) == len(p.command_list) leninstance = len(procedures.values()) lenstatic = len(Procedure.procedures) logging.debug("Procedure.procedures: %s" % Procedure.procedures) assert leninstance + init_proc_number == lenstatic, "different lengths: %s!=%s" %(leninstance, lenstatic)
parser.add_argument('-f', '--force', action='store_true', default=False, help="force ") args = parser.parse_args() report = time.strftime("%y%m%d", time.localtime(time.time())) logger.init(".", "avagent.log") from AVCommon.logger import logging globals()['logging']=logging logging.debug(args) mq = None procedure = None if args.procedure and args.procedure_file: logging.info("Procedure %s" % args.procedure) logging.debug("pwd: %s" % os.getcwd()) path = os.getcwd() procs = Procedure.load_from_file(args.procedure_file) logging.debug("%s" % procs) procedure = procs[args.procedure] mq = MQFeedProcedure() Procedure.check() try: avagent = AVAgent(args.vm, args.redis, args.session) avagent.start_agent(mq, procedure, force=args.force) except: logging.exception("FATAL")
def test_vm_commands(): yaml = """ TEST1: - START_VM TEST2: - EXECUTE_VM: c:\\users\\avtest\\desktop\\pubsub\\started.bat - PUSH: - [/tmp/gggg] - c:\\users\\avtest\\desktop - SCREENSHOT: /tmp/maggic_path.png TEST3: - PUSH: - [gggg, jojojo] - /tmp - c:\\users\\avtest\\desktop - PUSH: - [AVAgent/av_agent.py, AVAgent/build.py, AVAgent/package.py, AVAgent/rcs_client.py, AVCommon/commands/START_AGENT.py, AVCommon/commands/STOP_AGENT.py, AVCommon/commands/BUILD.py, AVCommon/commands/GET.py, AVCommon/commands/SET.py] - /home/olli/AVTest - c:\\AVTest - PULL: - [gggg, jojojo] - c:\\users\\avtest\\desktop - /tmp/cpl TEST4: - START_VM - SCREENSHOT: /tmp/magic_img_path.png - STOP_VM TEST5: - PUSH: - [AVCommon/commands/client/*.py] - /home/olli/AVTest - C:\\AVTest - PUSH: - [AVAgent/*.py] - /home/olli/AVTest - C:\\AVTest UPLOAD_AGENT: - PUSH: - [AVAgent/av_agent.py, AVAgent/build.py, AVAgent/package.py, AVAgent/rcs_client.py, AVCommon/commands/*.py] - /home/olli/AVTest - c:\\AVTest UPDATE: - REVERT - START_VM - SLEEP: 180 - CALL: UPLOAD_AGENT - INTERNET: True - SLEEP: 120 - INTERNET: False - STOP_VM - START_VM - SLEEP: 180 - STOP_VM - REFRESH_SNAPSHOT ZLEEP: - SLEEP: 120 T_IS: - CHECK_INFECTION - SLEEP: - 10 - 20 - CHECK_SHUTDOWN - SLEEP: 5 TEST_INTERNET: - INTERNET: True - SLEEP: 15 - INTERNET: False TEST_DIR: - PUSH: - [gigi/gggg] - /tmp - C:/Users/avtest/Desktop/gigi - SLEEP: 10 - DELETE_DIR: C:/Users/avtest/Desktop/gigi TEST_DIR_KO: - DELETE_DIR: C:/Users/avtest/Desktop/gigiol TEST_STOP1: - STOP_VM: 60 TEST_STOP: - STOP_VM """ procedures = Procedure.load_from_yaml(yaml) #vms = ["noav", "zenovm"] vms = ["zenoav"] redis_host = "localhost" mq = MQStar(redis_host) mq.clean() vm_manager.vm_conf_file = "../AVMaster/conf/vms.cfg" dispatcher = Dispatcher(mq, vms) ''' logging.info("STARTING TEST 1") dispatcher.dispatch(procedures["TEST1"]) import time time.sleep(200) logging.info("STARTING TEST 2") dispatcher.dispatch(procedures["TEST2"]) time.sleep(30) dispatcher.dispatch(procedures["TEST3"]) time.sleep(30) logging.info("STARTING TEST UPDATE PROCEDURE") dispatcher.dispatch(procedures["UPDATE"]) ''' logging.info("STARTING TEST 5") dispatcher.dispatch(procedures["TEST_STOP"])
def test_procedure_file(): procedures = Procedure.load_from_file("../AVMaster/conf/procedures/procedures.yaml") assert procedures, "empty procedures" logging.debug("procedures: %s" % procedures) for p in procedures.values(): assert isinstance(p, Procedure), "not a Procedure: %s" % p
def test_avagent_get_set(): host = "localhost" vms = [ "testvm_%d" % i for i in range(100) ] #command_client={ 'COMMAND_CLIENT': [{ 'SET': [ 'windows' 'whatever']}]} procedure = """ TEST: - START_AGENT - SET: {pippo: franco} - SET: backend: 192.168.100.201 frontend: 172.20.100.204 redis: 10.0.20.1 - SET: android: binary: {admin: false, demo: true} melt: {} platform: android sign: {} - GET: pippo - STOP_AGENT """ test = Procedure.load_from_yaml(procedure) host = "localhost" mq = MQStar(host) mq.clean() logging.debug("MQ session: %s" % mq.session) #istanzia n client e manda delle procedure. vm_manager.vm_conf_file = "../AVMaster/conf/vms.cfg" # dispatcher, inoltra e riceve i comandi della procedura test sulle vm dispatcher = Dispatcher(mq, vms, timeout = 10) thread = threading.Thread(target=dispatcher.dispatch, args=(test["TEST"],)) thread.start() #p = Process(target=dispatcher.dispatch, args=(test["TEST"],)) #p.start() # i client vengono eseguiti asincronicamente e comunicano tramite redis al server #pool = Pool(len(vms)) #r = pool.map_async(av_agent.start_agent, ( (v, host, mq.session) for v in vms) ) #r.get() #notare che i results dei client non ci interessano, viaggia tutto su channel/command. for v in vms: t = threading.Thread(target=av_agent.start_agent_args, args=(v, host, mq.session, True) ) t.start() #p = Process(target=av_agent.start_agent, args=( tuple([v, host, mq.session,]))) #p.start() # chiusura del server #p.join() thread.join() r = report.Report() logging.debug("sent: %s" % r.c_sent) logging.debug("received: %s" % r.c_received)