def _gotEnvironment(self, resdict):
info("Got environment %r", resdict)
self._environment = resdict
self.emit("start")
self._starttime = int(time.time())
self._storage.startNewTestRun(self, self._clientid)
self._runNextBatch()
def next(self):
if not self._initialized:
self._initialize()
if not self.globalidx < self.combinations:
raise StopIteration
# return the next dict of arguments
# contains a copy of all static arguments
# plus the next combination of generators
res = self.statics.copy()
if self.generators:
info("we have generators")
# extend with current generator values
for key in self.genlist:
info("key")
gen, idx = self.generators[key][:2]
# split generator name
keys = key.split(",")
if len(keys) > 1:
if isinstance(gen[idx],list):
gens = gen[idx]
else:
gens = gen[idx].split(",")
for i in range(len(keys)):
res[keys[i]] = gens[i]
else:
res[keys[0]] = gen[idx]
# update values
self._updateGeneratorsPosition()
# update global idx
self.globalidx += 1
return res
def tearDownVmethod(self):
info("uuid:%s", self.uuid)
# FIXME : tear down the other process gracefully
# by first sending it the termination remote signal
# and then checking it's killed
try:
self.callRemoteTearDown()
finally:
if self._testrun:
if self._newremotetestsid:
self._testrun.disconnect(self._newremotetestsid)
self._newremotetestsid = 0
if self._testrunremovedtestsigid:
self._testrun.disconnect(self._testrunremovedtestsigid)
self._testrunremovedtestsigid = 0
if self._processpollid:
gobject.source_remove(self._processpollid)
self._processpollid = 0
if self._process:
# double check it hasn't actually exited
# give the test up to one second to exit
if self._returncode is None:
self._returncode = utils.kill_process (self._process)
self._process = None
if self._redir_tty_thread is not None:
self._redir_tty_thread.exit()
self._redir_tty_thread = None
if not self._returncode is None:
info("Process returned %d", self._returncode)
self.extraInfo("subprocess-return-code", self._returncode)
self.validateChecklistItem("subprocess-exited-normally", self._returncode == 0)
def test(self):
info("uuid:%s proxy:%r", self.uuid, self._isproxy)
if self._isproxy:
self.callRemoteTest()
else:
# really do the test
raise Exception("I shouldn't be called ! I am the remote test !")
def remoteTearDown(self):
"""
Remote-side tearDown() method.
Subclasses wishing to clean up their tests or collect information to
send at the end, should implement this in their subclass and chain up
to the parent remoteTearDown() at the *beginning of their
implementation.
If the parent method returns False, return False straight-away
"""
if self._remote_tearing_down:
return False
self._remote_tearing_down = True
info("%s remoteTimeoutId:%r", self.uuid, self._remotetimeoutid)
if self.rusage_start:
usage = resource.getrusage(resource.RUSAGE_SELF)
rusage_end = (time.time(), usage.ru_utime, usage.ru_stime)
rusage_diffs = [end - start for start, end in zip(self.rusage_start, rusage_end)]
real_time, user_time, system_time = rusage_diffs
if real_time:
cpu_load = float(user_time + system_time) / real_time * 100.0
self.extraInfo("cpu-load", cpu_load)
# remote the timeout
if self._remotetimeoutid:
gobject.source_remove(self._remotetimeoutid)
self._remotetimedout = False
self._remotetimeoutid = 0
self.validateStep("no-timeout", not self._remotetimedout)
return True
def validateChecklistItem(self, checkitem, validated=True, description = None):
"""
Validate a checklist item in the checklist.
checkitem is one of the keys of __test_checklist__
validated is a boolean indicating whether that item should be
validated or not.
Called by the test itself
"""
info("checklist item %s for item %r : %r" % (checkitem, self, validated))
# check for valid checkitem
if not checkitem in self._possiblechecklist:
return
# check to see if we don't already have it
if checkitem in dict(self._checklist):
return
self._checklist.append((checkitem, bool(validated)))
if not validated:
if self.isExpectedFailure(checkitem, self._extrainfo):
self._expected_failures[checkitem] = True
if description and description != "":
explanation = description
else:
explanation = self.processFailure(checkitem, self._extrainfo)
if explanation is not None:
self._error_explanations[checkitem] = explanation
self.emit("check", checkitem, validated)
def stop(self):
info("uuid:%s proxy:%r", self.uuid, self._isproxy)
if self._isproxy:
Test.stop(self)
else:
self.tearDown()
self.remoteStopSignal()
def _start(self, glob):
desc = self.arguments.get("desc")
basename = self.arguments.get("outputfile-basename")
category = self.arguments.get("category")
files = []
paths = {}
if desc:
info("No path set, trying to use stdout and stding specific files")
if not 'stderr' in desc and not 'stdout' in desc:
warning("Neither of stdout-path, stderr-path and path specified"
"Can not use the monitor")
return False
if 'stderr' in desc:
stderr_file, stderr_path = self._getTempFiles(basename, "stderr", glob, category)
self.test.setStderr(stderr_file)
files.append(stderr_file)
paths["stderr-file"] = stderr_path
if 'stdout' in desc:
stdout_file, stdout_path = self._getTempFiles(basename, "stdout", glob, category)
self.test.setStdout(stdout_file)
files.append(stdout_file)
paths["stdout-file"] = stdout_path
else:
_file, path = self._getTempFiles(basename, "stdoutanderr", glob, category)
self.test.setStdOutAndErr(path)
files.append(_file)
paths["stdout-and-stderr-file"] = path
return files, paths
def _runNextBatch(self):
""" Runs the next test batch """
if len(self._tests) == 0:
# if nothing left, stop
info("No more tests batch to run, we're done")
self._stoptime = int(time.time())
self._storage.endTestRun(self)
self._running = False
self.emit("done")
return False
info("Getting next test batch")
# pop out the next batch
test, args, monitors = self._tests.pop(0)
self._currenttest = test
self._currentmonitors = monitors
self._currentarguments = args
info("Current test : %r" % test)
info("Current monitors : %r" % monitors)
info("Current arguments : %r" % args)
# and run the first one of that batch
self._runNext()
return False
def _stopMonitors(self):
for monitorinstance in self._monitorinstances:
if not monitorinstance.stop():
info("Could not stop monitor %s", monitorinstance)
continue
ofiles = monitorinstance.getIterationOutputFiles(self._iteration)
if ofiles:
self.iteration_outputfiles.update(ofiles)
def _singleTestDone(self, test):
info("Done with test %r , success rate %02f%%",
test, test.getSuccessPercentage())
self.emit("single-test-done", test)
# FIXME : Improvement : disconnect all signals from that test
if test in self._runninginstances:
self._runninginstances.remove(test)
self._storage.newTestFinished(self, test)
self._runNext()
def _removedRemoteTest(self, testrun, uuid):
if not uuid == self.uuid:
return
info("%s our remote counterpart has left", self.uuid)
# abort if the test hasn't actually finished
self._remoteinstance = None
if not self._stopping:
self.stop()
def remoteTest(self):
"""
Remote-side test() method.
Subclasses should implement this method and chain up to the parent
remoteTest() method at the *beginning* of their implementation.
"""
info("%s", self.uuid)
# add a timeout
self._remotetimeoutid = gobject.timeout_add(self._timeout * 1000, self._remoteTestTimeoutCb)
def _generate(self):
res = []
for path in self.paths:
fullpath = os.path.abspath(path)
if os.path.isfile(fullpath) and self._is_valid_file(fullpath):
res.append(fullpath)
else:
res.extend(self._get_files(fullpath))
info("Returning %d files" % len(res))
return res
def probe_test(self, filename):
if not "insanity-test-" in filename:
return False
info ('Running %s, which might be a test', filename)
try:
process = subprocess.Popen([filename, '--insanity-metadata'],
stdin = None, stdout = subprocess.PIPE, stderr = subprocess.PIPE,
universal_newlines=True)
except Exception,e:
info('Exception running process (%s), not a test', e)
return False
def kill_private_dbus():
"""
Kill the private dbus daemon used by the client
"""
global private_bus_pid, private_bus, private_bus_address
if private_bus_pid:
info("Killing private dbus daemon [pid:%d]" % int(private_bus_pid))
os.kill(int(private_bus_pid), signal.SIGKILL)
private_bus = None
private_bus_address = None
private_bus_pid = None
def _dbusNameOwnerChangedSignal(self, name, oldowner, newowner):
# we only care about connections named net.gstreamer.Insanity.Test.xxx
info("name:%s , oldowner:%s, newowner:%s" % (name, oldowner, newowner))
if not name.startswith("net.gstreamer.Insanity.Test.Test"):
return
# extract uuid
uuid = name.rsplit('.Test', 1)[-1]
if newowner == "":
self.emit("removed-remote-test", uuid)
elif oldowner == "":
self.emit("new-remote-test", uuid)
def validateChecklistItem(self, checkitem):
"""
Validate a checklist item in the checklist.
checkitem is one of the keys of __test_checklist__
Called by the test itself
"""
info("checklist item %s for item %r" % (checkitem, self))
if not checkitem in self._checklist:
return
self._checklist[checkitem] = True
def extraInfo(self, key, value):
"""
Give extra information obtained while running the tests.
If key was already given, the new value will override the value
previously given for the same key.
Called by the test itself
"""
info("uuid:%s, key:%s, value:%r", self.uuid, key, value)
self._extrainfo[key] = value
self.emit("extra-info", key, value)
def remoteSetUp(self):
"""
Remote-side setUp() method.
Subclasses should implement this method and chain up to the parent
remoteSetUp() method at the *end* of their implementation.
"""
info("%s", self.uuid)
usage = resource.getrusage(resource.RUSAGE_SELF)
self.rusage_start = (time.time(), usage.ru_utime, usage.ru_stime)
# if not overriden, we just emit the "ready" signal
self.remoteReadySignal()
def _voidRemoteStopCallBackHandler(self):
info("%s", self.uuid)
self.validateChecklistItem("subprocess-exited-normally")
self._prepareArguments()
Test.stop(self)
self.ping()
# Check if we have new arguments and
# have to run another time
if self.args:
self.start()
else:
self.tearDown()
def tearDown(self):
info("uuid:%s proxy:%r", self.uuid, self._isproxy)
if self._isproxy:
# FIXME : tear down the other process gracefully
# by first sending it the termination remote signal
# and then checking it's killed
try:
self.callRemoteStop()
finally:
if self._testrun:
if self._newremotetestsid:
self._testrun.disconnect(self._newremotetestsid)
self._newremotetestsid = 0
if self._testrunremovedtestsigid:
self._testrun.disconnect(self._testrunremovedtestsigid)
self._testrunremovedtestsigid = 0
if self._processpollid:
gobject.source_remove(self._processpollid)
self._processpollid = 0
if self._process:
# double check it hasn't actually exited
# give the test up to one second to exit
tries = 10
while self._returncode is None and not tries == 0:
time.sleep(0.1)
self._returncode = self._process.poll()
tries -= 1
if self._returncode is None:
info("Process isn't done yet, terminating it")
os.kill(self._process.pid, signal.SIGTERM)
time.sleep(1)
self._returncode = self._process.poll()
if self._returncode is None:
info("Process did not terminate, killing it")
os.kill(self._process.pid, signal.SIGKILL)
time.sleep(1)
self._returncode = self._process.poll()
if self._returncode is None:
# Probably turned into zombie process, something is
# really broken...
info("Process did not exit after SIGKILL")
self._process = None
if not self._returncode is None:
info("Process returned %d", self._returncode)
self.validateStep("subprocess-exited-normally", self._returncode == 0)
self.extraInfo("subprocess-return-code", self._returncode)
else:
self.remoteTearDown()
Test.tearDown(self)
def setUp(self):
info("uuid:%s", self.uuid)
if Test.setUp(self) == False:
return False
# get the remote launcher
pargs = self._preargs
pargs.extend(self.get_remote_launcher_args())
shell = isinstance (pargs, basestring)
cwd = self._testrun.getWorkingDirectory()
self._environ["PRIVATE_DBUS_ADDRESS"] = self._bus_address
info("Setting PRIVATE_DBUS_ADDRESS : %r" % self._bus_address)
info("bus:%r" % self._bus)
self._prepareArguments()
if False: # useful to allow some time to run dbus-monitor on the private bus
print("Setting PRIVATE_DBUS_ADDRESS : %r" % self._bus_address)
time.sleep(5)
# spawn the other process
info("opening %r" % pargs)
info("cwd %s" % cwd)
try:
self._subprocessspawntime = time.time()
self._process = subprocess.Popen(pargs,
stdin = self._stdin,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
env=self._environ,
shell = shell,
cwd=cwd)
self._ensureOutRedirection()
self._pid = self._process.pid
except:
exception("Error starting the subprocess command ! %r", pargs)
self.validateChecklistItem("dbus-process-spawned", False)
return False
debug("Subprocess created successfully [pid:%d]", self._pid)
self.validateChecklistItem("dbus-process-spawned")
# add a poller for the proces
self._processpollid = gobject.timeout_add(500, self._pollSubProcess)
# Don't forget to set a timeout for waiting for the connection
return True
def __init__(self, paths=[], recursive=True,
matching=[], reject=[], *args,
**kwargs):
"""
paths : list of paths and/or files
recursive : go down in subdirectories
matching : will only return files matching the given masks
reject : will not return files matching the given masks
"""
Generator.__init__(self, paths=paths, recursive=recursive,
matching=matching, reject=reject,
*args, **kwargs)
self.paths = paths
self.recursive = recursive
self.matching = matching
self.reject = reject
info("paths:%r, recursive:%r, matching:%r, reject:%r" % (paths, recursive, matching, reject))
def _newRemoteTest(self, testrun, uuid):
if not uuid == self.uuid:
return
info("%s our remote counterpart has started", self.uuid)
self.validateStep("dbus-process-connected")
self._subprocessconnecttime = time.time()
delay = self._subprocessconnecttime - self._subprocessspawntime
self.extraInfo("subprocess-spawn-time", delay)
# we need to give the remote process the following information:
# * filename where the Test class is located (self.get_file())
# * class name (self.__class__.__name__)
# * the arguments (self.arguments) + proxy=True
rname = "net.gstreamer.Insanity.Test.Test%s" % self.uuid
rpath = "/net/gstreamer/Insanity/Test/RemotePythonRunner%s" % self.uuid
# get the proxy object to our counterpart
remoteobj = self._bus.get_object(rname, rpath)
debug("Got remote runner object %r" % remoteobj)
# call createTestInstance()
remoterunner = dbus.Interface(remoteobj, "net.gstreamer.Insanity.RemotePythonRunner")
debug("Got remote iface %r" % remoterunner)
args = self.arguments.copy()
args["bus_address"] = self._bus_address
args["timeout"] = self._timeout
if self._outputfiles:
args["outputfiles"] = self.getOutputFiles()
debug(
"Creating remote instance with arguments %s %s %s %r",
self.get_file(),
self.__module__,
self.__class__.__name__,
args,
)
try:
remoterunner.createTestInstance(
self.get_file(),
self.__module__,
self.__class__.__name__,
args,
reply_handler=self._createTestInstanceCallBack,
error_handler=self._voidRemoteErrBackHandler,
)
except:
exception("Exception raised when creating remote instance !")
self.validateStep("remote-instanced-created", False)
self.stop()
def validateStep(self, checkitem, validated=True):
"""
Validate a step in the checklist.
checkitem is one of the keys of __test_checklist__
validated is a boolean indicating whether that step should be
validated or not.
Called by the test itself
"""
info("step %s for item %r : %r" % (checkitem, self, validated))
# check for valid checkitem
if not checkitem in self._possiblechecklist:
return
# check to see if we don't already have it
if checkitem in dict(self._checklist):
return
self._checklist.append((checkitem, bool(validated)))
# self._checklist[checkitem] = True
self.emit("check", checkitem, validated)
def addTest(self, test, arguments, monitors=None):
"""
Adds test with the given arguments (or generator) and monitors
to the list of tests to be run
monitors are a list of tuples containing:
* the monitor class
* (optional) the arguments to use on that monitor
"""
if not isinstance(test, type) and not issubclass(test, Test):
raise TypeError("Given test is not a Test object !")
# arguments NEED to be an Arguments object or a dictionnary
if isinstance(arguments, dict):
# convert dictionnary to Arguments
info("Creating Arguments for %r" % arguments)
arguments = Arguments(**arguments)
elif not isinstance(arguments, Arguments):
raise TypeError("Test arguments need to be of type Arguments or dict")
self._tests.append((test, arguments, monitors))
def __init__(self, singlerun=False, storage=None, *args, **kwargs):
dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
info("starting")
self._ml = gobject.MainLoop()
self._bus = dbustools.get_private_session_bus()
self._busname = dbus.service.BusName("net.gstreamer.TesterClient", self._bus)
# dbus.service.Object.__init__(self, dbustools.get_private_session_bus(), "/here")
dbus.service.Object.__init__(self, self._bus, "/here")
self._testruns = []
self._storage = None
self._clientid = None
if storage:
self.setStorage(storage)
# _current is the current TestRun being executed
self._current = None
# _running is True if the mainloop is running
self._running = False
# If _singlerun == True, the client will quit after
# all testruns are done
self._singlerun = singlerun
def setUp(self):
info("uuid:%s proxy:%r", self.uuid, self._isproxy)
if Test.setUp(self) == False:
return False
if self._isproxy:
# get the remote launcher
pargs = self._preargs
pargs.extend(self.get_remote_launcher_args())
cwd = self._testrun.getWorkingDirectory()
self._environ["PRIVATE_DBUS_ADDRESS"] = self._bus_address
info("Setting PRIVATE_DBUS_ADDRESS : %r" % self._bus_address)
info("bus:%r" % self._bus)
# spawn the other process
info("opening %r" % pargs)
info("cwd %s" % cwd)
try:
self._subprocessspawntime = time.time()
self._process = subprocess.Popen(
pargs, stdin=self._stdin, stdout=self._stdout, stderr=self._stderr, env=self._environ, cwd=cwd
)
self._pid = self._process.pid
except:
exception("Error starting the subprocess command ! %r", pargs)
self.validateStep("dbus-process-spawned", False)
return False
debug("Subprocess created successfully [pid:%d]", self._pid)
self.validateStep("dbus-process-spawned")
# add a poller for the proces
self._processpollid = gobject.timeout_add(500, self._pollSubProcess)
# Don't forget to set a timeout for waiting for the connection
else:
# remote instance setup
# self.remoteSetUp()
pass
return True
def _newRemoteTest(self, testrun, uuid):
if not uuid == self.uuid:
return
info("%s our remote counterpart has started", self.uuid)
self.validateChecklistItem("dbus-process-connected")
self._subprocessconnecttime = time.time()
delay = self._subprocessconnecttime - self._subprocessspawntime
self.extraInfo("subprocess-spawn-time", int(delay * 1000))
# we need to give the remote process the following information:
# * filename where the Test class is located (self.get_file())
# * class name (self.__class__.__name__)
# * the arguments (self.arguments)
rname = "net.gstreamer.Insanity.Test.Test%s" % self.uuid
rpath = "/net/gstreamer/Insanity/Test/Test%s" % self.uuid
# get the proxy object to our counterpart
remoteobj = self._bus.get_object(rname, rpath)
debug("Got remote runner object %r" % remoteobj)
# call createTestInstance()
remoterunner = dbus.Interface(remoteobj,
"net.gstreamer.Insanity.Test")
debug("Got remote iface %r" % remoterunner)
try:
delay = time.time() - self._subprocessconnecttime
self._remoteinstance = dbus.Interface(remoteobj,
"net.gstreamer.Insanity.Test")
info ('Listening to signals from %s' % self._remoteinstance)
self._remoteinstance.connect_to_signal("remoteDoneSignal",
self._remoteDoneCb)
self._remoteinstance.connect_to_signal("remoteValidateChecklistItemSignal",
self._remoteValidateChecklistItemCb)
self._remoteinstance.connect_to_signal("remoteExtraInfoSignal",
self._remoteExtraInfoCb)
self._remoteinstance.connect_to_signal("remotePingSignal",
self._remotePingCb)
self.callRemoteSetUp()
except:
exception("Exception raised when creating remote instance !")
self.stop()
