def uninstall(backend):
logging.debug("- Clean Server: %s" % (backend))
connection.host = backend
target = get_target_name()
logging.debug("target: %s" % (target))
with connection() as client:
logging.debug("connected")
operation_id, group_id = client.operation(connection.operation)
targets = client.targets(operation_id, target)
if len(targets) != 1:
return False, "not one target: %s" % len(targets)
target_id = targets[0]
instances = client.instances_by_target_id(target_id)
#logging.debug("found these instances: %s" % instances)
if len(instances) != 1:
logging.warn("more than one instance")
for instance in instances:
instance_id = instance['_id']
target_id = instance['path'][1]
#logging.debug('closing instance: %s' % instance)
client.instance_close(instance_id)
return True, "Instance closed"
def on_answer(vm, success, answer):
from AVMaster import vm_manager
#assert command.context, "Null context"
logging.debug("CROP answer: %s|%s" % (success, answer))
# answer = [1,5,7]
if answer and isinstance(answer, list):
logging.warn("We have to PULL images: %s" % answer)
dir = "%s/crop" % logger.logdir
for iter in answer:
try:
src = "%s/%s.png" % (config.basedir_crop, iter)
#name = src.split('/')[-1]
dst_dir = "%s/%s" % (dir, vm)
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
dst = "%s/%s.png" % (dst_dir, iter)
src = src.replace('/', '\\')
logging.debug("PULL: %s -> %s" % (src, dst))
vm_manager.execute(vm, "copyFileFromGuest", src, dst)
except:
logging.exception("Cannot get image %s" % src)
def uninstall(backend):
logging.debug("- Clean Server: %s" % (backend))
connection.host = backend
target = get_target_name()
logging.debug("target: %s" % (target))
with connection() as client:
logging.debug("connected")
operation_id, group_id = client.operation(connection.operation)
targets = client.targets(operation_id, target)
if len(targets) != 1:
return False, "not one target: %s" % len(targets)
target_id = targets[0]
instances = client.instances_by_target_id(target_id)
#logging.debug("found these instances: %s" % instances)
if len(instances) != 1:
logging.warn("more than one instance")
for instance in instances:
instance_id = instance['_id']
target_id = instance['path'][1]
#logging.debug('closing instance: %s' % instance)
client.instance_close(instance_id)
return True, "Instance closed"
def on_answer(vm, success, answer):
from AVMaster import vm_manager
#assert command.context, "Null context"
logging.debug("CROP answer: %s|%s" % (success, answer))
# answer = [1,5,7]
if answer and isinstance(answer, list):
logging.warn("We have to PULL images: %s" % answer)
dir = "%s/crop" % logger.logdir
for iter in answer:
try:
src = "%s/%s.png" % (config.basedir_crop, iter)
#name = src.split('/')[-1]
dst_dir = "%s/%s" %(dir, vm)
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
dst = "%s/%s.png" % (dst_dir, iter)
src = src.replace('/','\\')
logging.debug("PULL: %s -> %s" % (src, dst))
vm_manager.execute(vm, "copyFileFromGuest", src ,dst)
except:
logging.exception("Cannot get image %s" % src)
def execute_scout(self):
""" build and execute the """
factory_id, ident, exe = self.execute_pull()
new_exe = "build\\scout.exe"
logging.debug("execute_scout: %s" % exe)
shutil.copy(exe[0], new_exe)
self._execute_build(exe)
if self.kind == "melt": # and not exploit
sleep(60)
executed = self.execute_agent_startup()
if not executed:
logging.warn("did'n executed")
add_result("+ WARN did not drop startup")
logging.debug("- Scout, Wait for 5 minutes: %s" % time.ctime())
sleep(300)
for tries in range(1, 10):
logging.debug("- Scout, Trigger sync for 30 seconds, try %s" %
tries)
self._trigger_sync(timeout=30)
logging.debug("- Scout, wait for 1 minute: %s" % time.ctime())
sleep(60 * 1)
instance_id = self.check_instance(ident)
if instance_id:
break
for i in range(10):
self._click_mouse(100 + i, 0)
if not instance_id:
add_result("+ FAILED SCOUT SYNC")
output = self._list_processes()
logging.debug(output)
else:
self.check_level(instance_id, "scout")
if self.kind == "melt":
try:
found = False
for d, b in itertools.product(start_dirs, names):
filename = "%s/%s.exe" % (d, b)
filename = filename.replace("/", "\\")
if os.path.exists(filename):
found = True
if not found:
logging.warn("did'n executed")
add_result("+ FAILED NO STARTUP")
except:
pass
logging.debug("- Result: %s" % instance_id)
return instance_id
def execute_scout(self):
""" build and execute the """
factory_id, ident, exe = self.execute_pull()
new_exe = "build\\scout.exe"
logging.debug("execute_scout: %s" % exe)
shutil.copy(exe[0], new_exe)
self._execute_build(exe)
if self.kind == "melt": # and not exploit
sleep(60)
executed = self.execute_agent_startup()
if not executed:
logging.warn("did'n executed")
add_result("+ WARN did not drop startup")
logging.debug("- Scout, Wait for 5 minutes: %s" % time.ctime())
sleep(300)
for tries in range(1, 10):
logging.debug("- Scout, Trigger sync for 30 seconds, try %s" % tries)
self._trigger_sync(timeout=30)
logging.debug("- Scout, wait for 1 minute: %s" % time.ctime())
sleep(60 * 1)
instance_id = self.check_instance(ident)
if instance_id:
break
for i in range(10):
self._click_mouse(100 + i, 0)
if not instance_id:
add_result("+ FAILED SCOUT SYNC")
output = self._list_processes()
logging.debug(output)
else:
self.check_level(instance_id, "scout")
if self.kind == "melt":
try:
found = False
for d,b in itertools.product(start_dirs,names):
filename = "%s/%s.exe" % (d,b)
filename = filename.replace("/","\\")
if os.path.exists(filename):
found = True
if not found:
logging.warn("did'n executed")
add_result("+ FAILED NO STARTUP")
except:
pass
logging.debug("- Result: %s" % instance_id)
return instance_id
def check():
ret = True
called = set()
system = []
try:
for name, p in Procedure.procedures.items():
for c in p.command_list:
if c.name == "CALL":
call = c.args
called.add(call)
if call not in Procedure.procedures.keys():
logging.error(
"Error in procedure: %s, call to non existant proc: %s"
% (name, call))
ret = False
if c.name == "REPORT":
calls = c.args
for c in calls:
call = None
if isinstance(c, basestring):
call = c
elif isinstance(c, dict):
call = c.keys()[0]
else:
logging.error(
"Error in procedure: %s, call to non compliant proc: %s"
% (name, call))
if call:
called.add(call)
if call not in Procedure.procedures.keys():
logging.error(
"Error in procedure: %s, call to non existant proc: %s"
% (name, call))
ret = False
procs = set(Procedure.procedures.keys())
for p in called:
if p.startswith("SYSTEM_"):
logging.warn("system proc called: %s" % p)
for p in procs.difference(called):
good_start = ["TEST", "SYSTEM", "UPDATE"]
if not any([p.startswith(g) for g in good_start]):
logging.warn("probably unused PROC: %s" % p)
else:
system.append(p)
system.sort()
logging.info("Callable Procedures: %s" % system)
return ret
except:
logging.exception("Check")
def _create_new_factory(self, operation, target, factory, config):
with connection() as c:
assert c
if not c.logged_in():
logging.warn("Not logged in")
logging.debug("DBG type: " + self.ftype + ", operation: " +
operation + ", target: " + target + ", factory: " +
factory)
operation_id, group_id = c.operation(operation)
if not operation_id:
raise RuntimeError("Cannot get operations")
# gets all the target with our name in an operation
targets = c.targets(operation_id, target)
if len(targets) > 0:
# keep only one target
for t in targets[1:]:
c.target_delete(t)
target_id = targets[0]
agents = c.agents(target_id)
for agent_id, ident, name in agents:
logging.debug("DBG %s %s %s" % (agent_id, ident, name))
if name.startswith(factory):
logging.debug("- Delete instance: %s %s" %
(ident, name))
c.instance_delete(agent_id)
else:
logging.debug("- Create target: %s" % target)
target_id = c.target_create(
operation_id, target,
'made by vmavtest at %s' % time.ctime())
factory_id, ident = c.factory_create(
operation_id, target_id, self.ftype, factory,
'made by vmavtestat at %s' % time.ctime())
with open(config) as f:
conf = f.read()
conf = re.sub(r'"host": ".*"', r'"host": "%s"' % self.host[1],
conf)
#logging.debug("conf: %s" % conf)
c.factory_add_config(factory_id, conf)
with open('build/config.actual.json', 'wb') as f:
f.write(conf)
return (target_id, factory_id, ident)
def _create_new_factory(self, operation, target, factory, config):
with connection() as c:
assert c
if not c.logged_in():
logging.warn("Not logged in")
logging.debug(
"DBG type: " + self.ftype + ", operation: " + operation + ", target: " + target + ", factory: " + factory)
operation_id, group_id = c.operation(operation)
if not operation_id:
raise RuntimeError("Cannot get operations")
# gets all the target with our name in an operation
targets = c.targets(operation_id, target)
if len(targets) > 0:
# keep only one target
for t in targets[1:]:
c.target_delete(t)
target_id = targets[0]
agents = c.agents(target_id)
for agent_id, ident, name in agents:
logging.debug("DBG %s %s %s" % (agent_id, ident, name))
if name.startswith(factory):
logging.debug("- Delete instance: %s %s" % (ident, name))
c.instance_delete(agent_id)
else:
logging.debug("- Create target: %s" % target)
target_id = c.target_create(
operation_id, target, 'made by vmavtest at %s' % time.ctime())
factory_id, ident = c.factory_create(
operation_id, target_id, self.ftype, factory,
'made by vmavtestat at %s' % time.ctime()
)
with open(config) as f:
conf = f.read()
conf = re.sub(
r'"host": ".*"', r'"host": "%s"' % self.host[1], conf)
#logging.debug("conf: %s" % conf)
c.factory_add_config(factory_id, conf)
with open('build/config.actual.json', 'wb') as f:
f.write(conf)
return (target_id, factory_id, ident)
def check():
ret = True
called = set()
system = []
try:
for name,p in Procedure.procedures.items():
for c in p.command_list:
if c.name == "CALL":
call = c.args
called.add(call)
if call not in Procedure.procedures.keys():
logging.error("Error in procedure: %s, call to non existant proc: %s" % (name, call))
ret = False
if c.name == "REPORT":
calls = c.args
for c in calls:
call = None
if isinstance(c, basestring):
call = c
elif isinstance(c, dict):
call = c.keys()[0]
else:
logging.error("Error in procedure: %s, call to non compliant proc: %s" % (name, call))
if call:
called.add(call)
if call not in Procedure.procedures.keys():
logging.error("Error in procedure: %s, call to non existant proc: %s" % (name, call))
ret = False
procs = set(Procedure.procedures.keys())
for p in called:
if p.startswith("SYSTEM_"):
logging.warn("system proc called: %s" % p)
for p in procs.difference(called):
good_start=["TEST","SYSTEM","UPDATE"]
if not any([p.startswith(g) for g in good_start]) :
logging.warn("probably unused PROC: %s" % p)
else:
system.append(p)
system.sort()
logging.info("Callable Procedures: %s" % system)
return ret
except:
logging.exception("Check")
def execute_agent(args, level, platform):
""" starts the vm and execute elite,scout or pull, depending on the level """
global internet_checked
ftype = args.platform_type
logging.debug("DBG ftype: %s" % ftype)
vmavtest = AgentBuild(args.backend, args.frontend, platform, args.kind,
ftype, args.blacklist, args.soldierlist, args.param,
args.puppet, args.asset_dir, args.factory)
""" starts a scout """
if socket.gethostname().lower() not in args.nointernetcheck:
if not internet_checked and internet_on():
add_result("+ ERROR: I reach Internet")
return False
internet_checked = True
logging.debug("- Network unreachable")
logging.debug("- Server: %s/%s %s" %
(args.backend, args.frontend, args.kind))
if platform == "exploit_web":
vmavtest.execute_web_expl(args.frontend)
else:
if vmavtest.create_user_machine():
#add_result("+ SUCCESS USER CONNECT")
if vmavtest.server_errors():
#add_result("+ WARN SERVER ERRORS")
logging.warn("Server errors")
#add_result("+ SUCCESS SERVER CONNECT")
action = {
"elite": vmavtest.execute_elite,
"scout": vmavtest.execute_scout,
"pull": vmavtest.execute_pull,
"elite_fast": vmavtest.execute_elite_fast,
"soldier_fast": vmavtest.execute_soldier_fast,
"soldier": vmavtest.execute_soldier
}
sleep(5)
action[level]()
else:
add_result("+ ERROR USER CREATE")
return True
def start(self):
self.load_procedures()
proc = Procedure.procedures[self.procedure]
assert proc, "cannot find the specified procedure: %s" % self.procedure
# command line vm list overrides procedures.yaml
if self.vm_names==[''] and proc.command_list and proc.command_list[0].name.startswith("VM"):
vm_command = proc.command_list.pop(0)
self.vm_names = vm_command.execute('server', None, vm_command.args)[1]
logging.info("VM override: %s" % self.vm_names)
assert self.vm_names, "No VM specified"
mq = MQStar(self.args.redis, self.args.session)
if self.args.clean:
logging.warn("cleaning mq")
mq.clean()
logging.info("mq session: %s" % mq.session)
dispatcher = Dispatcher(mq, self.vm_names)
dispatcher.dispatch(proc, pool = self.pool)
def execute_agent(args, level, platform):
""" starts the vm and execute elite,scout or pull, depending on the level """
global internet_checked
ftype = args.platform_type
logging.debug("DBG ftype: %s" % ftype)
vmavtest = AgentBuild(args.backend, args.frontend,
platform, args.kind, ftype, args.blacklist, args.soldierlist, args.param, args.puppet, args.asset_dir, args.factory)
""" starts a scout """
if socket.gethostname().lower() not in args.nointernetcheck:
if not internet_checked and internet_on():
add_result("+ ERROR: I reach Internet")
return False
internet_checked = True
logging.debug("- Network unreachable")
logging.debug("- Server: %s/%s %s" % (args.backend, args.frontend, args.kind))
if platform == "exploit_web":
vmavtest.execute_web_expl(args.frontend)
else:
if vmavtest.create_user_machine():
#add_result("+ SUCCESS USER CONNECT")
if vmavtest.server_errors():
#add_result("+ WARN SERVER ERRORS")
logging.warn("Server errors")
#add_result("+ SUCCESS SERVER CONNECT")
action = {"elite": vmavtest.execute_elite, "scout": vmavtest.execute_scout,
"pull": vmavtest.execute_pull, "elite_fast": vmavtest.execute_elite_fast,
"soldier_fast": vmavtest.execute_soldier_fast, "soldier": vmavtest.execute_soldier }
sleep(5)
action[level]()
else:
add_result("+ ERROR USER CREATE")
return True
def execute(vm, protocol, args):
""" server side """
from AVMaster import vm_manager
logging.debug(" CS PUSH: %s" % str(args))
assert vm, "null self.vm"
assert isinstance(args, list)
if isinstance(args[0], list):
src_files, src_dir, dst_dir = args
elif isinstance(args[0], basestring):
src_files, src_dir, dst_dir = args, config.basedir_server, config.basedir_av
else:
raise RuntimeError("wrong arguments")
assert isinstance(src_files, list), "PUSH expects a list of src files"
all_src = []
relative_parents = set()
""" look if i need all files in one directory """
for src_file in src_files:
g = glob.glob(os.path.join(src_dir, src_file))
if not g:
logging.warn("Empty glob")
# if you arrive here, then you already found the file on the filesystem.
# typically the file have a relative path
for f in g:
# s is the relative file, expanded by glob
s = f.replace("%s/" % src_dir, "")
all_src.append(s)
# logging.debug("file completo f: %s, file relativo s: %s" % (f,s))
# logging.debug("Check if exists file %s" % f)
assert os.path.exists(f), "%s %s" % (f, os.getcwd())
# inserito da Marco
# logging.debug("Check if exists file %s" % os.path.join(src_dir, s))
assert os.path.exists(os.path.join(src_dir, s)), "%s %s" % (s, os.getcwd())
# add all the parents to the relative_parents set, to avoid repetitions
p = os.path.split(s)[0]
while p and p != src_dir:
relative_parents.add(p)
# print("1_relative parents")
p = os.path.split(p)[0]
# sorts the parents by length, so that parent is always before its sons
relative_parents.add("./")
parents = list(relative_parents)
parents.sort(lambda x, y: len(x) - len(y))
logging.debug("parents: %s" % parents)
ntdir = lambda x: x.replace("/", "\\")
print(parents)
for r in parents:
# print(dst_dir)
rdir = ntdir(os.path.join(dst_dir, r))
# print("2_crea cartelle %s" % rdir)
logging.debug("mkdir %s " % (rdir))
vm_manager.execute(vm, "mkdirInGuest", rdir)
""" then upload parsed files """
logging.debug("All files to copy are:\n%s" % src_files)
if not all_src:
return False, "Empty file list"
for src_file in all_src:
# print("3_processa file")
src = os.path.join(src_dir, src_file)
dst = ntdir(os.path.join(dst_dir, src_file))
# logging.debug("Check if exists file %s" % src)
if not os.path.exists(src):
return False, "Not existent file: %s" % src
else:
pass
# logging.debug("file exists")
logging.debug("%s copy %s -> %s" % (vm, src, dst))
r = vm_manager.execute(vm, "copyFileToGuest", src, dst)
if r > 0:
return False, "Cant Copy %s on VM" % src_file
return True, "Files copied on VM"
def normalize(data):
""" a command cane be unserialized in many ways:
- command instance
- dict: { command: payload }
- tuple/array: (cmd, success, payload) or (cmd, payload)
- str: "(cmd, success, payload)"
payload is evaluated via ast, so that it can contain a type like tuple, array, number, dict and so on
if payload begins with a "|", it's considered a plain string and it's not evaluated
"""
#ident, command, answer = serialized.split(',', 2)
#assert(ident == "CMD")
cmd = data
success = None
args = None
result = None
vm = None
assert data, "cannot normalize a null argument"
identified = "instance"
if isinstance(data, Command):
logging.warn("normalizing a command")
return data.name, data.success, data.args, data.result, data.vm
elif isinstance(data, dict):
identified = "dict"
assert len(data) == 1
cmd = data.keys()[0]
args = data[cmd]
elif not isinstance(data, str) and len(data) == 3:
identified = "len 3"
cmd, success, payload = data
if success == None:
args = payload
else:
result = payload
elif isinstance(data, str):
identified = "str"
m = re.compile("\('(\w+)\', (\w+), (.+)\)").match(data)
if m:
identified = "reg"
groups = m.groups()
assert len(groups) == 3
cmd = groups[0]
success = ast.literal_eval(groups[1])
try:
payload = ast.literal_eval(groups[2])
except SyntaxError:
payload = groups[2]
except ValueError:
payload = groups[2]
if success == None:
args = payload
else:
result = payload
#logging.debug(1)Command.knownCommands
if config.verbose:
logging.debug("identified: %s" % identified)
assert isinstance(success,
bool) or success is None, "success: %s" % success
assert isinstance(cmd, str), "not a string: %s" % cmd
return (cmd, success, args, result, vm)
def dispatch(self, procedure, pool=0 ):
global received
exit = False
command.context = {}
procedure.add_begin_end()
#logging.debug("- SERVER len(procedure): %s" % len(procedure))
self.num_commands = len(procedure)
report.init(procedure.name)
assert self.vms
assert self.vms[0], "please specify at least one VM"
logging.debug("self.vms: %s" % self.vms)
av_machines = OrderedDict()
p_id = 0
for vm in self.vms:
av_machines[vm] = Protocol(self, vm, procedure, id = p_id)
p_id += 1
if pool == 0:
pool = len(self.vms)
Protocol.pool = pool
self.pool_start(av_machines.values(), pool)
self.ended = set()
answered = 0
while not exit and len(self.ended) < len(self.vms):
rec = self.mq.receive_server(blocking=True, timeout=self.timeout)
if rec is not None:
c, msg = rec
try:
command_unserialize = command.unserialize(msg)
except:
logging.exception("cannot unserialize: %s" % msg)
#exit = True
continue
#command_unserialize =
logging.info("- RECEIVED %s, %s" % (c, red(command_unserialize)))
if c not in av_machines.keys():
logging.warn("A message for %s probably belongs to another test!" % c)
continue
p = av_machines[c]
try:
answer = p.receive_answer(c, command_unserialize)
except:
logging.exception("cannot receive: %s" % command_unserialize)
continue
report.received(c, command_unserialize)
if answer.success == None:
#logging.debug("- SERVER IGNORING")
continue
answered += 1
#logging.debug("- SERVER RECEIVED ANSWER: %s" % answer.success)
if answer.name == "END":
self.end(c)
logging.info("- RECEIVE END: %s, %s" % (c, self.ended))
logging.debug("self.ended: (%s/%s) %s" % (len(self.ended), len(self.vms), self.ended))
elif p.on_error != "DISABLED" and (answer.success or p.on_error == "CONTINUE"):
r = p.send_next_command()
cmd = p.last_command
report.sent(p.vm, cmd)
logging.info("- SENT: %s, %s" % (c, red(cmd)))
if not r:
logging.info("- SENDING ERROR, ENDING: %s" %c)
self.end(c)
logging.debug("self.ended: (%s/%s) %s" % (len(self.ended), len(self.vms), self.ended))
else:
# answer.success == False
# deve skippare fino al command: END_PROC
if p.on_error == "SKIP":
logging.debug("on_error == %s" % p.on_error)
r = p.send_next_call()
cmd = p.last_command
if cmd:
report.sent(p.vm, cmd)
else:
logging.info("- RECEIVE ERROR, ENDING: %s" %c)
self.end(c)
logging.debug("self.ended: (%s/%s) %s" % (len(self.ended), len(self.vms), self.ended))
elif p.on_error == "DISABLED":
logging.debug("on_error == DISABLED")
r = p.send_next_proc()
cmd = p.last_command
if cmd:
report.sent(p.vm, cmd)
else:
logging.info("- RECEIVE ERROR, ENDING: %s" %c)
self.end(c)
logging.debug("self.ended: (%s/%s) %s" % (len(self.ended), len(self.vms), self.ended))
else:
assert p.on_error == "STOP"
logging.info("- RECEIVE ERROR, STOP: %s" %c)
self.end(c)
logging.debug("self.ended: (%s/%s) %s" % (len(self.ended), len(self.vms), self.ended))
else:
logging.info("- SERVER RECEIVED empty")
exit = True
report.finish()
logging.debug("answered: %s, ended: %s, num_commands: %s" % ( answered, len(self.ended), self.num_commands))
assert len(self.ended) == len(self.vms), "answered: %s, ended: %s, num_commands: %s" % ( answered, len(self.ended), len(self.vms))
#assert answered >= (len(self.vms) * (self.num_commands)), "answered: %s, len(vms): %s, num_commands: %s" % (answered , len(self.vms), self.num_commands)
return answered
def execute(vm, protocol, args):
""" server side """
from AVMaster import vm_manager
logging.debug(" CS PUSH: %s" % str(args))
assert vm, "null self.vm"
assert isinstance(args, list)
if isinstance(args[0], list):
src_files, src_dir, dst_dir = args
elif isinstance(args[0], basestring):
src_files, src_dir, dst_dir = args, config.basedir_server, config.basedir_av
else:
raise RuntimeError("wrong arguments")
assert isinstance(src_files, list), "PUSH expects a list of src files"
all_src = []
relative_parents = set()
""" look if i need all files in one directory """
for src_file in src_files:
g = glob.glob(os.path.join(src_dir, src_file))
if not g:
logging.warn("Empty glob")
# if you arrive here, then you already found the file on the filesystem.
# typically the file have a relative path
for f in g:
# s is the relative file, expanded by glob
s = f.replace("%s/" % src_dir, "")
all_src.append(s)
#logging.debug("file completo f: %s, file relativo s: %s" % (f,s))
#logging.debug("Check if exists file %s" % f)
assert os.path.exists(f), "%s %s" % (f, os.getcwd())
#inserito da Marco
#logging.debug("Check if exists file %s" % os.path.join(src_dir, s))
assert os.path.exists(os.path.join(src_dir,
s)), "%s %s" % (s, os.getcwd())
# add all the parents to the relative_parents set, to avoid repetitions
p = os.path.split(s)[0]
while p and p != src_dir:
relative_parents.add(p)
#print("1_relative parents")
p = os.path.split(p)[0]
# sorts the parents by length, so that parent is always before its sons
relative_parents.add("./")
parents = list(relative_parents)
parents.sort(lambda x, y: len(x) - len(y))
logging.debug("parents: %s" % parents)
ntdir = lambda x: x.replace("/", "\\")
print(parents)
for r in parents:
#print(dst_dir)
rdir = ntdir(os.path.join(dst_dir, r))
#print("2_crea cartelle %s" % rdir)
logging.debug("mkdir %s " % (rdir))
vm_manager.execute(vm, "mkdirInGuest", rdir)
""" then upload parsed files """
logging.debug("All files to copy are:\n%s" % src_files)
if not all_src:
return False, "Empty file list"
for src_file in all_src:
#print("3_processa file")
src = os.path.join(src_dir, src_file)
dst = ntdir(os.path.join(dst_dir, src_file))
#logging.debug("Check if exists file %s" % src)
if not os.path.exists(src):
return False, "Not existent file: %s" % src
else:
pass
#logging.debug("file exists")
logging.debug("%s copy %s -> %s" % (vm, src, dst))
r = vm_manager.execute(vm, "copyFileToGuest", src, dst)
if r > 0:
return False, "Cant Copy %s on VM" % src_file
return True, "Files copied on VM"
def execute(vm, protocol, args):
""" server side """
from AVMaster import vm_manager
logging.debug(" CS PUSHZIP: %s" % str(args))
assert vm, "null self.vm"
assert isinstance(args, list)
if isinstance(args[0], basestring):
src_files, src_dir, dst_dir = args, config.basedir_server, config.basedir_av
else:
raise RuntimeError("wrong arguments")
assert isinstance(src_files, list), "PUSHZIP expects a list of src files"
all_src = []
""" look if i need all files in one directory """
for src_file in src_files:
g = glob.glob(os.path.join(src_dir, src_file))
if not g:
logging.warn("Empty glob")
# if you arrive here, then you already found the file on the filesystem.
# typically the file have a relative path
for f in g:
# s is the relative file, expanded by glob
s = f.replace("%s/" % src_dir, "")
all_src.append(s)
#logging.debug("Check if exists file %s" % f)
assert os.path.exists(f), "%s %s" % (f, os.getcwd())
#inserito da Marco
#logging.debug("Check if exists file %s" % os.path.join(src_dir, s))
assert os.path.exists(os.path.join(src_dir,
s)), "%s %s" % (s, os.getcwd())
ntdir = lambda x: x.replace("/", "\\")
print 'creating archive'
d = tempfile.mkdtemp()
zfname = d + '/zipfile_write.zip'
zf = zipfile.ZipFile(zfname, mode='w')
pwd = config.basedir_server
""" then upload parsed files """
logging.debug("All files to copy are:\n%s" % src_files)
if not all_src:
return False, "Empty file list"
for src_file in all_src:
#print("3_processa file")
src = os.path.join(src_dir, src_file)
#logging.debug("Check if exists file %s" % src)
if not os.path.exists(src):
return False, "Not existent file: %s" % src
else:
pass
logging.debug("%s adding %s -> %s" % (vm, src_file, src))
zf.write(src_file)
zf.close()
#zip file is ready
vm_manager.execute(vm, "mkdirInGuest", ntdir(dst_dir))
# copy unzip (it should be already in AVAgent/assets...)
unzipexe = "assets/unzip.exe"
dst = ntdir(os.path.join(dst_dir, "unzip.exe"))
logging.debug("Copy unzip: %s -> %s" % (unzipexe, dst))
vm_manager.execute(vm, "copyFileToGuest", unzipexe, dst)
tmpzip = "tmp.zip"
dst = ntdir(os.path.join(dst_dir, tmpzip))
logging.debug("Copy zip: %s -> %s" % (zfname, dst))
vm_manager.execute(vm, "copyFileToGuest", zfname, dst)
logging.debug("Executing unzip on %s" % dst)
unzipargs = ("/AVTest/unzip.exe", ["-o", "-d", "c:\\avtest",
dst], 40, True, True)
ret = vm_manager.execute(vm, "executeCmd", *unzipargs)
logging.debug("ret: %s" % ret)
logging.debug("Removing zip: %s" % d)
shutil.rmtree(d)
return True, "Files copied on VM"
def normalize(data):
""" a command cane be unserialized in many ways:
- command instance
- dict: { command: payload }
- tuple/array: (cmd, success, payload) or (cmd, payload)
- str: "(cmd, success, payload)"
payload is evaluated via ast, so that it can contain a type like tuple, array, number, dict and so on
if payload begins with a "|", it's considered a plain string and it's not evaluated
"""
#ident, command, answer = serialized.split(',', 2)
#assert(ident == "CMD")
cmd = data
success = None
args = None
result = None
vm = None
assert data, "cannot normalize a null argument"
identified = "instance"
if isinstance(data, Command):
logging.warn("normalizing a command")
return data.name, data.success, data.args, data.result, data.vm
elif isinstance(data, dict):
identified = "dict"
assert len(data) == 1
cmd = data.keys()[0]
args = data[cmd]
elif not isinstance(data, str) and len(data) == 3:
identified = "len 3"
cmd, success, payload = data
if success == None:
args = payload
else:
result = payload
elif isinstance(data, str):
identified = "str"
m = re.compile("\('(\w+)\', (\w+), (.+)\)").match(data)
if m:
identified = "reg"
groups = m.groups()
assert len(groups) == 3
cmd = groups[0]
success = ast.literal_eval(groups[1])
try:
payload = ast.literal_eval(groups[2])
except SyntaxError:
payload = groups[2]
except ValueError:
payload = groups[2]
if success == None:
args = payload
else:
result = payload
#logging.debug(1)Command.knownCommands
if config.verbose:
logging.debug("identified: %s" % identified)
assert isinstance(success, bool) or success is None, "success: %s" % success
assert isinstance(cmd, str), "not a string: %s" % cmd
return (cmd, success, args, result, vm)
def execute(vm, protocol, args):
""" server side """
from AVMaster import vm_manager
logging.debug(" CS PUSHZIP: %s" % str(args))
assert vm, "null self.vm"
assert isinstance(args, list)
if isinstance(args[0], basestring):
src_files, src_dir, dst_dir = args, config.basedir_server, config.basedir_av
else:
raise RuntimeError("wrong arguments")
assert isinstance(src_files, list), "PUSHZIP expects a list of src files"
all_src = []
""" look if i need all files in one directory """
for src_file in src_files:
g = glob.glob(os.path.join(src_dir, src_file))
if not g:
logging.warn("Empty glob")
# if you arrive here, then you already found the file on the filesystem.
# typically the file have a relative path
for f in g:
# s is the relative file, expanded by glob
s = f.replace("%s/" % src_dir, "")
all_src.append(s)
#logging.debug("Check if exists file %s" % f)
assert os.path.exists(f), "%s %s" % (f, os.getcwd())
#inserito da Marco
#logging.debug("Check if exists file %s" % os.path.join(src_dir, s))
assert os.path.exists(os.path.join(src_dir, s)), "%s %s" % (s, os.getcwd())
ntdir = lambda x: x.replace("/", "\\")
print 'creating archive'
d = tempfile.mkdtemp()
zfname = d + '/zipfile_write.zip'
zf = zipfile.ZipFile(zfname, mode='w')
pwd = config.basedir_server
""" then upload parsed files """
logging.debug("All files to copy are:\n%s" % src_files)
if not all_src:
return False, "Empty file list"
for src_file in all_src:
#print("3_processa file")
src = os.path.join(src_dir, src_file)
#logging.debug("Check if exists file %s" % src)
if not os.path.exists(src):
return False, "Not existent file: %s" % src
else:
pass
logging.debug("%s adding %s -> %s" % (vm, src_file, src))
zf.write(src_file)
zf.close()
#zip file is ready
vm_manager.execute(vm, "mkdirInGuest", ntdir(dst_dir))
# copy unzip (it should be already in AVAgent/assets...)
unzipexe = "assets/unzip.exe"
dst = ntdir(os.path.join(dst_dir, "unzip.exe"))
logging.debug("Copy unzip: %s -> %s" % (unzipexe, dst) )
vm_manager.execute(vm, "copyFileToGuest", unzipexe, dst)
tmpzip = "tmp.zip"
dst = ntdir(os.path.join(dst_dir, tmpzip))
logging.debug("Copy zip: %s -> %s" % (zfname, dst) )
vm_manager.execute(vm, "copyFileToGuest", zfname, dst)
logging.debug("Executing unzip on %s" % dst)
unzipargs= ( "/AVTest/unzip.exe", [ "-o", "-d", "c:\\avtest", dst], 40, True, True )
ret = vm_manager.execute(vm, "executeCmd", *unzipargs )
logging.debug("ret: %s" % ret)
logging.debug("Removing zip: %s" % d)
shutil.rmtree(d)
return True, "Files copied on VM"
def dispatch(self, procedure, pool=0):
global received
exit = False
command.context = {}
procedure.add_begin_end()
#logging.debug("- SERVER len(procedure): %s" % len(procedure))
self.num_commands = len(procedure)
report.init(procedure.name)
assert self.vms
assert self.vms[0], "please specify at least one VM"
logging.debug("self.vms: %s" % self.vms)
av_machines = OrderedDict()
p_id = 0
for vm in self.vms:
av_machines[vm] = Protocol(self, vm, procedure, id=p_id)
p_id += 1
if pool == 0:
pool = len(self.vms)
Protocol.pool = pool
self.pool_start(av_machines.values(), pool)
self.ended = set()
answered = 0
while not exit and len(self.ended) < len(self.vms):
rec = self.mq.receive_server(blocking=True, timeout=self.timeout)
if rec is not None:
c, msg = rec
try:
command_unserialize = command.unserialize(msg)
except:
logging.exception("cannot unserialize: %s" % msg)
#exit = True
continue
#command_unserialize =
logging.info("- RECEIVED %s, %s" %
(c, red(command_unserialize)))
if c not in av_machines.keys():
logging.warn(
"A message for %s probably belongs to another test!" %
c)
continue
p = av_machines[c]
try:
answer = p.receive_answer(c, command_unserialize)
except:
logging.exception("cannot receive: %s" %
command_unserialize)
continue
report.received(c, command_unserialize)
if answer.success == None:
#logging.debug("- SERVER IGNORING")
continue
answered += 1
#logging.debug("- SERVER RECEIVED ANSWER: %s" % answer.success)
if answer.name == "END":
self.end(c)
logging.info("- RECEIVE END: %s, %s" % (c, self.ended))
logging.debug("self.ended: (%s/%s) %s" %
(len(self.ended), len(self.vms), self.ended))
elif p.on_error != "DISABLED" and (answer.success or p.on_error
== "CONTINUE"):
r = p.send_next_command()
cmd = p.last_command
report.sent(p.vm, cmd)
logging.info("- SENT: %s, %s" % (c, red(cmd)))
if not r:
logging.info("- SENDING ERROR, ENDING: %s" % c)
self.end(c)
logging.debug(
"self.ended: (%s/%s) %s" %
(len(self.ended), len(self.vms), self.ended))
else:
# answer.success == False
# deve skippare fino al command: END_PROC
if p.on_error == "SKIP":
logging.debug("on_error == %s" % p.on_error)
r = p.send_next_call()
cmd = p.last_command
if cmd:
report.sent(p.vm, cmd)
else:
logging.info("- RECEIVE ERROR, ENDING: %s" % c)
self.end(c)
logging.debug(
"self.ended: (%s/%s) %s" %
(len(self.ended), len(self.vms), self.ended))
elif p.on_error == "DISABLED":
logging.debug("on_error == DISABLED")
r = p.send_next_proc()
cmd = p.last_command
if cmd:
report.sent(p.vm, cmd)
else:
logging.info("- RECEIVE ERROR, ENDING: %s" % c)
self.end(c)
logging.debug(
"self.ended: (%s/%s) %s" %
(len(self.ended), len(self.vms), self.ended))
else:
assert p.on_error == "STOP"
logging.info("- RECEIVE ERROR, STOP: %s" % c)
self.end(c)
logging.debug(
"self.ended: (%s/%s) %s" %
(len(self.ended), len(self.vms), self.ended))
else:
logging.info("- SERVER RECEIVED empty")
exit = True
report.finish()
logging.debug("answered: %s, ended: %s, num_commands: %s" %
(answered, len(self.ended), self.num_commands))
assert len(self.ended) == len(
self.vms), "answered: %s, ended: %s, num_commands: %s" % (
answered, len(self.ended), len(self.vms))
#assert answered >= (len(self.vms) * (self.num_commands)), "answered: %s, len(vms): %s, num_commands: %s" % (answered , len(self.vms), self.num_commands)
return answered