def load_procedures(self):
if os.path.exists("AVMaster/conf/procedures.yaml"):
Procedure.load_from_file("AVMaster/conf/procedures.yaml")
confs = glob.glob("AVMaster/conf/procedures/*.yaml")
for conf in confs:
logging.info("Loading conf: %s" % conf)
Procedure.load_from_file(conf)
if not Procedure.check():
raise SyntaxError("Errors in procedures")
def test_dispatcher():
c = command.factory(["BEGIN", True, ['whatever', 'end']])
agentFiles = ""
params = ""
update = Procedure("UPDATE", ["REVERT", "START_VM", "RELOG", "STOP_VM"])
dispatch = Procedure("DISPATCH", ["REVERT", "START_VM", ("PUSH", None, agentFiles)])
scout = Procedure("SCOUT", [
("CALL", None, "dispatch"),
("PUSH", None, agentFiles),
])
assert update
assert dispatch
assert scout
def test_procedure_insert():
c = command.factory(["BEGIN", True, ['whatever', 'end']])
agentFiles = ""
params = ""
p1 = Procedure("UPDATE", ["REVERT", "START_VM", "RELOG", "STOP_VM"])
p2 = Procedure("DISPATCH", ["REVERT", "START_VM", ("PUSH", None, agentFiles)])
lp1= len(p1)
lp2= len(p2)
p1.insert(p2)
assert p1
assert p2
assert len(p1) == lp1 + lp2
def notest_ProtocolEval():
host = "localhost"
mq = MQStar(host)
mq.clean()
c = "client1"
mq.add_client(c)
commands = [
"BEGIN", ("EVAL_SERVER", "dir()"), ("EVAL_SERVER", "locals()"),
("EVAL_SERVER", "__import__('os').getcwd()"), ("END", None, None)
]
procedure = Procedure("PROC", commands)
p = Protocol(mq, c, procedure)
while p.send_next_command():
logging.debug("sent command")
exit = False
while not exit:
rec = mq.receive_server(blocking=True, timeout=10)
if rec is not None:
logging.debug("- SERVER RECEIVED %s %s" % (rec, type(rec)))
c, msg = rec
answer = p.receive_answer(c, msg)
logging.debug("- SERVER RECEIVED ANSWER: ", answer.success)
if answer.name == "END" or not answer.success:
logging.debug("- SERVER RECEIVE END")
#if answer.success:
a = """('client1', ('EVAL_SERVER', True, {'self': <Command_EVAL_SERVER.Command_EVAL_SERVER object at 0x10931f810>, 'args': 'locals()'}))""" # p.send_next_command()
else:
logging.debug("- SERVER RECEIVED empty")
exit = True
def test_ProtocolCall():
host = "localhost"
mq = MQStar(host)
mq.clean()
c = "client1"
mq.add_client(c)
yaml = """
HELLO:
- EVAL_SERVER: 3+2
BASIC:
- EVAL_SERVER: dir()
- CALL: HELLO
CALLER:
- BEGIN
- CALL: BASIC
- EVAL_SERVER: locals()
- END
"""
procedures = Procedure.load_from_yaml(yaml)
caller = Procedure.procedures["CALLER"]
basic = Procedure.procedures["BASIC"]
class D:
pass
d = D()
d.mq = mq
p = Protocol(d, c, caller)
while p.send_next_command():
logging.debug("sent command")
exit = False
answers =0
while not exit:
rec = mq.receive_server(blocking=True, timeout=10)
if rec is not None:
logging.debug("- SERVER RECEIVED %s %s" % (rec, type(rec)))
c, msg = rec
command_unserialize = command.unserialize(msg)
answer = p.receive_answer(c, command_unserialize)
logging.debug("- SERVER RECEIVED ANSWER: %s" % answer.success)
if answer.success:
answers += 1
if answer.name == "END" or not answer.success:
logging.debug("- SERVER RECEIVE END")
#if answer.success:
exit = True
else:
logging.debug("- SERVER RECEIVED empty")
exit = True
assert answers == 7, "wrong answers: %s" % answers
def test_avagent_pull():
host = "localhost"
vms = ["testvm_%d" % i for i in range(1)]
#command_client={ 'COMMAND_CLIENT': [{ 'SET': [ 'windows' 'whatever']}]}
procedure = """
TEST:
- START_AGENT
- SET:
- [backend, 192.168.100.201]
- [frontend, 172.20.100.204]
- [redis, 10.0.20.1]
- BUILD: [ pull, windows, silent]
- STOP_AGENT
"""
test = Procedure.load_from_yaml(procedure)
host = "localhost"
mq = MQStar(host)
mq.clean()
logging.debug("MQ session: %s" % mq.session)
#istanzia n client e manda delle procedure.
vm_manager.vm_conf_file = "../AVMaster/conf/vms.cfg"
report = Report()
# dispatcher, inoltra e riceve i comandi della procedura test sulle vm
dispatcher = Dispatcher(mq, vms, report)
thread = threading.Thread(target=dispatcher.dispatch,
args=(test["TEST"], ))
thread.start()
#p = Process(target=dispatcher.dispatch, args=(test["TEST"],))
#p.start()
# i client vengono eseguiti asincronicamente e comunicano tramite redis al server
pool = Pool(len(vms))
r = pool.map_async(av_agent.start_agent,
((v, host, mq.session) for v in vms))
r.get(
) #notare che i results dei client non ci interessano, viaggia tutto su channel/command.
# chiusura del server
#p.join()
thread.join()
logging.debug(dispatcher.report)
logging.debug("sent: %s" % dispatcher.report.c_sent)
logging.debug("received: %s" % Report.c_received)
def test_report_meta():
yaml = """
P1:
- SLEEP: 1
P2:
- SLEEP
P3:
- SLEEP: 2
P4:
- SLEEP
TEST_REPORT:
- VM: [noav,zenoav]
- SET_SERVER:
mail_recipients: [[email protected]]
- REPORT:
- P1: ["AVtest", "MyCase"]
- P2: ["AVtest", "MyOtherCase", INVERSE ]
- P3
- P4
"""
procedures = Procedure.load_from_yaml(yaml)
vms = ["noav", "zenovm"]
#vms = ["noav"]
redis_host = "localhost"
mq = MQStar(redis_host)
mq.clean()
vm_manager.vm_conf_file = "../AVMaster/conf/vms.cfg"
dispatcher = Dispatcher(mq, vms)
logging.info("STARTING TEST REPORT")
dispatcher.dispatch(procedures["TEST_REPORT"])
logging.info("STOPPING TEST REPORT")
r = report.Report()
#report.finish()
assert r
assert len(r.c_received) == len(vms)
assert len(r.c_sent) == len(vms)
assert len(r.current_procedure) == len(vms)
assert len(r.c_received) == len(vms), len(r.c_received)
for vm in vms:
assert vm in r.c_received
assert len(r.c_received[vm]) >= 18
def test_report_meta():
yaml = """
P1:
- SLEEP: 1
P2:
- SLEEP
P3:
- SLEEP: 2
P4:
- SLEEP
TEST_REPORT:
- VM: [noav,zenoav]
- SET_SERVER:
mail_recipients: [[email protected]]
- REPORT:
- P1: ["AVtest", "MyCase"]
- P2: ["AVtest", "MyOtherCase", INVERSE ]
- P3
- P4
"""
procedures = Procedure.load_from_yaml(yaml)
vms = ["noav", "zenovm"]
# vms = ["noav"]
redis_host = "localhost"
mq = MQStar(redis_host)
mq.clean()
vm_manager.vm_conf_file = "../AVMaster/conf/vms.cfg"
dispatcher = Dispatcher(mq, vms)
logging.info("STARTING TEST REPORT")
dispatcher.dispatch(procedures["TEST_REPORT"])
logging.info("STOPPING TEST REPORT")
r = report.Report()
# report.finish()
assert r
assert len(r.c_received) == len(vms)
assert len(r.c_sent) == len(vms)
assert len(r.current_procedure) == len(vms)
assert len(r.c_received) == len(vms), len(r.c_received)
for vm in vms:
assert vm in r.c_received
assert len(r.c_received[vm]) >= 18
def test_avagent_pull():
host = "localhost"
vms = [ "testvm_%d" % i for i in range(1) ]
#command_client={ 'COMMAND_CLIENT': [{ 'SET': [ 'windows' 'whatever']}]}
procedure = """
TEST:
- START_AGENT
- SET:
- [backend, 192.168.100.201]
- [frontend, 172.20.100.204]
- [redis, 10.0.20.1]
- BUILD: [ pull, windows, silent]
- STOP_AGENT
"""
test = Procedure.load_from_yaml(procedure)
host = "localhost"
mq = MQStar(host)
mq.clean()
logging.debug("MQ session: %s" % mq.session)
#istanzia n client e manda delle procedure.
vm_manager.vm_conf_file = "../AVMaster/conf/vms.cfg"
report= Report()
# dispatcher, inoltra e riceve i comandi della procedura test sulle vm
dispatcher = Dispatcher(mq, vms, report)
thread = threading.Thread(target=dispatcher.dispatch, args=(test["TEST"],))
thread.start()
#p = Process(target=dispatcher.dispatch, args=(test["TEST"],))
#p.start()
# i client vengono eseguiti asincronicamente e comunicano tramite redis al server
pool = Pool(len(vms))
r = pool.map_async(av_agent.start_agent, ( (v, host, mq.session) for v in vms) )
r.get() #notare che i results dei client non ci interessano, viaggia tutto su channel/command.
# chiusura del server
#p.join()
thread.join()
logging.debug(dispatcher.report)
logging.debug("sent: %s" % dispatcher.report.c_sent)
logging.debug("received: %s" % Report.c_received)
def test_dispatcher_server():
host = "localhost"
vms = ["noav", "zenovm"]
host = "localhost"
mq = MQStar(host)
mq.clean()
#istanzia n client e manda delle procedure.
vm_manager.vm_conf_file = "../AVMaster/conf/vms.cfg"
dispatcher = Dispatcher(mq, vms)
test = Procedure("TEST", [("EVAL_SERVER", None, 'vm'),
("SLEEP", None, 10)])
dispatcher.dispatch(test)
def execute(vm, protocol, mon_args):
""" client side, returns (bool,*) """
logging.debug(" SET %s" % str(mon_args))
assert vm, "null vm"
assert command.context is not None
assert isinstance(mon_args, list), "VM expects a list"
assert protocol
assert protocol.procedure
logging.debug("insert report init")
command_list = []
command_list.append("REPORT_INIT")
for proc_token in mon_args:
report_args = []
if isinstance(proc_token, basestring):
proc_name = proc_token
elif isinstance(proc_token, dict):
assert len(proc_token.keys()) == 1
proc_name = proc_token.keys()[0]
report_args = proc_token[proc_name]
else:
return False, "Error parsing"
logging.debug("insert report kind: %s args: %s" %
(proc_name, report_args))
command_list.append(["REPORT_KIND_INIT", None, (proc_name)])
command_list.append(["CALL", None, (proc_name)])
command_list.append(
["REPORT_KIND_END", None, (proc_name, report_args)])
command_list.append("REPORT_END")
proc = Procedure("_REPORT", command_list)
protocol.procedure.insert(proc)
#logging.debug("procedure: %s" % (protocol.procedure.command_list))
#logging.debug("report items: %s" % (command.context))
return True, "REPORT"
def test_procedure_yaml():
yaml = """UPDATE:
- REVERT
- START_VM
- RELOG
- STOP_VM
DISPATCH:
- REVERT
- START_VM
- RELOG
- PUSH:
- file.sh
- anotherfile.sh
SCOUT:
- CALL: DISPATCH
- START_AGENT
- BUILD
"""
init_proc_number = len(Procedure.procedures)
procedures = Procedure.load_from_yaml(yaml)
assert procedures, "empty procedures"
logging.debug("procedures: %s" % procedures)
assert len(procedures) == 3, "wrong procedures number: %s" % len(procedures)
for p in procedures.values():
assert isinstance(p, Procedure), "not a Procedure: %s" % p
assert p.name
assert p.command_list
assert len(p) == len(p.command_list)
leninstance = len(procedures.values())
lenstatic = len(Procedure.procedures)
logging.debug("Procedure.procedures: %s" % Procedure.procedures)
assert leninstance + init_proc_number == lenstatic, "different lengths: %s!=%s" %(leninstance, lenstatic)
parser.add_argument('-f', '--force', action='store_true', default=False,
help="force ")
args = parser.parse_args()
report = time.strftime("%y%m%d", time.localtime(time.time()))
logger.init(".", "avagent.log")
from AVCommon.logger import logging
globals()['logging']=logging
logging.debug(args)
mq = None
procedure = None
if args.procedure and args.procedure_file:
logging.info("Procedure %s" % args.procedure)
logging.debug("pwd: %s" % os.getcwd())
path = os.getcwd()
procs = Procedure.load_from_file(args.procedure_file)
logging.debug("%s" % procs)
procedure = procs[args.procedure]
mq = MQFeedProcedure()
Procedure.check()
try:
avagent = AVAgent(args.vm, args.redis, args.session)
avagent.start_agent(mq, procedure, force=args.force)
except:
logging.exception("FATAL")
def test_vm_commands():
yaml = """
TEST1:
- START_VM
TEST2:
- EXECUTE_VM: c:\\users\\avtest\\desktop\\pubsub\\started.bat
- PUSH:
- [/tmp/gggg]
- c:\\users\\avtest\\desktop
- SCREENSHOT: /tmp/maggic_path.png
TEST3:
- PUSH:
- [gggg, jojojo]
- /tmp
- c:\\users\\avtest\\desktop
- PUSH:
- [AVAgent/av_agent.py, AVAgent/build.py, AVAgent/package.py, AVAgent/rcs_client.py,
AVCommon/commands/START_AGENT.py, AVCommon/commands/STOP_AGENT.py,
AVCommon/commands/BUILD.py, AVCommon/commands/GET.py, AVCommon/commands/SET.py]
- /home/olli/AVTest
- c:\\AVTest
- PULL:
- [gggg, jojojo]
- c:\\users\\avtest\\desktop
- /tmp/cpl
TEST4:
- START_VM
- SCREENSHOT: /tmp/magic_img_path.png
- STOP_VM
TEST5:
- PUSH:
- [AVCommon/commands/client/*.py]
- /home/olli/AVTest
- C:\\AVTest
- PUSH:
- [AVAgent/*.py]
- /home/olli/AVTest
- C:\\AVTest
UPLOAD_AGENT:
- PUSH:
- [AVAgent/av_agent.py, AVAgent/build.py, AVAgent/package.py, AVAgent/rcs_client.py, AVCommon/commands/*.py]
- /home/olli/AVTest
- c:\\AVTest
UPDATE:
- REVERT
- START_VM
- SLEEP: 180
- CALL: UPLOAD_AGENT
- INTERNET: True
- SLEEP: 120
- INTERNET: False
- STOP_VM
- START_VM
- SLEEP: 180
- STOP_VM
- REFRESH_SNAPSHOT
ZLEEP:
- SLEEP: 120
T_IS:
- CHECK_INFECTION
- SLEEP:
- 10
- 20
- CHECK_SHUTDOWN
- SLEEP: 5
TEST_INTERNET:
- INTERNET: True
- SLEEP: 15
- INTERNET: False
TEST_DIR:
- PUSH:
- [gigi/gggg]
- /tmp
- C:/Users/avtest/Desktop/gigi
- SLEEP: 10
- DELETE_DIR: C:/Users/avtest/Desktop/gigi
TEST_DIR_KO:
- DELETE_DIR: C:/Users/avtest/Desktop/gigiol
TEST_STOP1:
- STOP_VM: 60
TEST_STOP:
- STOP_VM
"""
procedures = Procedure.load_from_yaml(yaml)
#vms = ["noav", "zenovm"]
vms = ["zenoav"]
redis_host = "localhost"
mq = MQStar(redis_host)
mq.clean()
vm_manager.vm_conf_file = "../AVMaster/conf/vms.cfg"
dispatcher = Dispatcher(mq, vms)
'''
logging.info("STARTING TEST 1")
dispatcher.dispatch(procedures["TEST1"])
import time
time.sleep(200)
logging.info("STARTING TEST 2")
dispatcher.dispatch(procedures["TEST2"])
time.sleep(30)
dispatcher.dispatch(procedures["TEST3"])
time.sleep(30)
logging.info("STARTING TEST UPDATE PROCEDURE")
dispatcher.dispatch(procedures["UPDATE"])
'''
logging.info("STARTING TEST 5")
dispatcher.dispatch(procedures["TEST_STOP"])
def test_vm_commands():
yaml = """
TEST1:
- START_VM
TEST2:
- EXECUTE_VM: c:\\users\\avtest\\desktop\\pubsub\\started.bat
- PUSH:
- [/tmp/gggg]
- c:\\users\\avtest\\desktop
- SCREENSHOT: /tmp/maggic_path.png
TEST3:
- PUSH:
- [gggg, jojojo]
- /tmp
- c:\\users\\avtest\\desktop
- PUSH:
- [AVAgent/av_agent.py, AVAgent/build.py, AVAgent/package.py, AVAgent/rcs_client.py,
AVCommon/commands/START_AGENT.py, AVCommon/commands/STOP_AGENT.py,
AVCommon/commands/BUILD.py, AVCommon/commands/GET.py, AVCommon/commands/SET.py]
- /home/olli/AVTest
- c:\\AVTest
- PULL:
- [gggg, jojojo]
- c:\\users\\avtest\\desktop
- /tmp/cpl
TEST4:
- START_VM
- SCREENSHOT: /tmp/magic_img_path.png
- STOP_VM
TEST5:
- PUSH:
- [AVCommon/commands/client/*.py]
- /home/olli/AVTest
- C:\\AVTest
- PUSH:
- [AVAgent/*.py]
- /home/olli/AVTest
- C:\\AVTest
UPLOAD_AGENT:
- PUSH:
- [AVAgent/av_agent.py, AVAgent/build.py, AVAgent/package.py, AVAgent/rcs_client.py, AVCommon/commands/*.py]
- /home/olli/AVTest
- c:\\AVTest
UPDATE:
- REVERT
- START_VM
- SLEEP: 180
- CALL: UPLOAD_AGENT
- INTERNET: True
- SLEEP: 120
- INTERNET: False
- STOP_VM
- START_VM
- SLEEP: 180
- STOP_VM
- REFRESH_SNAPSHOT
ZLEEP:
- SLEEP: 120
T_IS:
- CHECK_INFECTION
- SLEEP:
- 10
- 20
- CHECK_SHUTDOWN
- SLEEP: 5
TEST_INTERNET:
- INTERNET: True
- SLEEP: 15
- INTERNET: False
TEST_DIR:
- PUSH:
- [gigi/gggg]
- /tmp
- C:/Users/avtest/Desktop/gigi
- SLEEP: 10
- DELETE_DIR: C:/Users/avtest/Desktop/gigi
TEST_DIR_KO:
- DELETE_DIR: C:/Users/avtest/Desktop/gigiol
TEST_STOP1:
- STOP_VM: 60
TEST_STOP:
- STOP_VM
"""
procedures = Procedure.load_from_yaml(yaml)
#vms = ["noav", "zenovm"]
vms = ["zenoav"]
redis_host = "localhost"
mq = MQStar(redis_host)
mq.clean()
vm_manager.vm_conf_file = "../AVMaster/conf/vms.cfg"
dispatcher = Dispatcher(mq, vms)
'''
logging.info("STARTING TEST 1")
dispatcher.dispatch(procedures["TEST1"])
import time
time.sleep(200)
logging.info("STARTING TEST 2")
dispatcher.dispatch(procedures["TEST2"])
time.sleep(30)
dispatcher.dispatch(procedures["TEST3"])
time.sleep(30)
logging.info("STARTING TEST UPDATE PROCEDURE")
dispatcher.dispatch(procedures["UPDATE"])
'''
logging.info("STARTING TEST 5")
dispatcher.dispatch(procedures["TEST_STOP"])
def test_procedure_file():
procedures = Procedure.load_from_file("../AVMaster/conf/procedures/procedures.yaml")
assert procedures, "empty procedures"
logging.debug("procedures: %s" % procedures)
for p in procedures.values():
assert isinstance(p, Procedure), "not a Procedure: %s" % p
def test_avagent_get_set():
host = "localhost"
vms = [ "testvm_%d" % i for i in range(100) ]
#command_client={ 'COMMAND_CLIENT': [{ 'SET': [ 'windows' 'whatever']}]}
procedure = """
TEST:
- START_AGENT
- SET: {pippo: franco}
- SET:
backend: 192.168.100.201
frontend: 172.20.100.204
redis: 10.0.20.1
- SET:
android:
binary: {admin: false, demo: true}
melt: {}
platform: android
sign: {}
- GET: pippo
- STOP_AGENT
"""
test = Procedure.load_from_yaml(procedure)
host = "localhost"
mq = MQStar(host)
mq.clean()
logging.debug("MQ session: %s" % mq.session)
#istanzia n client e manda delle procedure.
vm_manager.vm_conf_file = "../AVMaster/conf/vms.cfg"
# dispatcher, inoltra e riceve i comandi della procedura test sulle vm
dispatcher = Dispatcher(mq, vms, timeout = 10)
thread = threading.Thread(target=dispatcher.dispatch, args=(test["TEST"],))
thread.start()
#p = Process(target=dispatcher.dispatch, args=(test["TEST"],))
#p.start()
# i client vengono eseguiti asincronicamente e comunicano tramite redis al server
#pool = Pool(len(vms))
#r = pool.map_async(av_agent.start_agent, ( (v, host, mq.session) for v in vms) )
#r.get() #notare che i results dei client non ci interessano, viaggia tutto su channel/command.
for v in vms:
t = threading.Thread(target=av_agent.start_agent_args, args=(v, host, mq.session, True) )
t.start()
#p = Process(target=av_agent.start_agent, args=( tuple([v, host, mq.session,])))
#p.start()
# chiusura del server
#p.join()
thread.join()
r = report.Report()
logging.debug("sent: %s" % r.c_sent)
logging.debug("received: %s" % r.c_received)