class PitiviTestsManager(TestsManager):
name = "pitivi"
_scenarios = ScenarioManager()
def __init__(self):
super(PitiviTestsManager, self).__init__()
def init(self):
self.fixme("Implement init checking")
return True
def add_options(self, parser):
group = parser.add_argument_group(
"Pitivi specific option group"
" and behaviours", description="")
group.add_argument("--pitivi-scenarios-paths",
dest="pitivi_scenario_paths",
default=os.path.join(os.path.basename(__file__),
"pitivi", "pitivi scenarios"),
help="Paths in which to look for scenario files")
def set_settings(self, options, args, reporter):
TestsManager.set_settings(self, options, args, reporter)
self._scenarios.config = self.options
try:
os.makedirs(utils.url2path(options.dest)[0])
except OSError:
pass
def list_tests(self):
return self.tests
def register_defaults(self):
scenarios = list()
for path in self.options.pitivi_scenario_paths:
for root, dirs, files in os.walk(path):
for f in files:
if not f.endswith(".scenario"):
continue
scenarios.append(os.path.join(path, root, f))
for scenario_name in scenarios:
scenario = self._scenarios.get_scenario(scenario_name)
if scenario is None:
continue
classname = "pitivi.%s" % (scenario.name)
self.add_test(
PitiviTest(classname,
self.options,
self.reporter,
scenario=scenario))
def main(libsdir):
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter,
prog='gst-validate-launcher', description=HELP)
parser.add_argument('testsuites', metavar='N', nargs='*',
help="""Lets you specify a file where the testsuite to execute is defined.
In the module if you want to work with a specific test manager(s) (for example,
'ges' or 'validate'), you should define the TEST_MANAGER variable in the
testsuite file (it can be a list of test manager names)
In this file you should implement a setup_tests function. That function takes
a TestManager and the GstValidateLauncher option as parameters and return True
if it succeeded loading the tests, False otherwise.
You will be able to configure the TestManager with its various methods. This
function will be called with each TestManager usable, for example you will be
passed the 'validate' TestManager in case the GstValidateManager launcher is
avalaible. You should configure it using:
* test_manager.add_scenarios: which allows you to register a list of scenario names to be run
* test_manager.set_default_blacklist: Lets you set a list of tuple of the form:
(@regex_defining_blacklister_test_names, @reason_for_the_blacklisting)
* test_manager.add_generators: which allows you to register a list of #GstValidateTestsGenerator
to be used to generate tests
* test_manager.add_encoding_formats:: which allows you to register a list #MediaFormatCombination to be used for transcoding tests
You can also set default values with:
* test_manager.register_defaults: Sets default values for all parametters
* test_manager.register_default_test_generators: Sets default values for the TestsGenerators to be used
* test_manager.register_default_scenarios: Sets default values for the scenarios to be executed
* test_manager.register_default_encoding_formats: Sets default values for the encoding formats to be tested
Note that all testsuite should be inside python modules, so the directory should contain a __init__.py file
""",
default=["validate", "ges"])
parser.add_argument("-d", "--debug", dest="debug",
action="store_true",
help="Let user debug the process on timeout")
parser.add_argument("-f", "--forever", dest="forever",
action="store_true",
help="Keep running tests until one fails")
parser.add_argument("-F", "--fatal-error", dest="fatal_error",
action="store_true",
help="Stop on first fail")
parser.add_argument("-t", "--wanted-tests", dest="wanted_tests",
action="append",
help="Define the tests to execute, it can be a regex."
" If it contains defaults_only, only default scenarios"
" will be executed")
parser.add_argument("-b", "--blacklisted-tests", dest="blacklisted_tests",
action="append",
help="Define the tests not to execute, it can be a regex.")
parser.add_argument("-L", "--list-tests",
dest="list_tests",
action="store_true",
help="List tests and exit")
parser.add_argument("-m", "--mute", dest="mute",
action="store_true",
help="Mute playback output, which means that we use "
"a fakesink")
parser.add_argument("-n", "--no-color", dest="no_color",
action="store_true",
help="Set it to output no colored text in the terminal")
parser.add_argument("-g", "--generate-media-info", dest="generate_info",
action="store_true",
help="Set it in order to generate the missing .media_infos files")
parser.add_argument("--update-media-info", dest="update_media_info",
action="store_true",
help="Set it in order to update exising .media_infos files")
parser.add_argument(
"-G", "--generate-media-info-with-frame-detection", dest="generate_info_full",
action="store_true",
help="Set it in order to generate the missing .media_infos files. "
"It implies --generate-media-info but enabling frame detection")
parser.add_argument("-lt", "--long-test-limit", dest="long_limit",
action='store',
help="Defines the limite from which a test is concidered as long (in seconds)"
" not that 0 will enable all tests", type=int),
parser.add_argument("-c", "--config", dest="config",
help="This is DEPRECATED, prefer using the testsuite format"
" to configure testsuites")
dir_group = parser.add_argument_group(
"Directories and files to be used by the launcher")
parser.add_argument('--xunit-file', action='store',
dest='xunit_file', metavar="FILE",
help=("Path to xml file to store the xunit report in. "
"Default is LOGSDIR/xunit.xml"))
dir_group.add_argument("-M", "--main-dir", dest="main_dir",
help="Main directory where to put files. Default is %s" % DEFAULT_MAIN_DIR)
dir_group.add_argument("--testsuites-dir", dest="testsuites_dir",
help="Directory where to look for testsuites. Default is %s"
" Note that GstValidate expect testsuite file to have .testsuite"
" as an extension in this folder." % DEFAULT_TESTSUITES_DIR)
dir_group.add_argument("-o", "--output-dir", dest="output_dir",
help="Directory where to store logs and rendered files. Default is MAIN_DIR")
dir_group.add_argument("-l", "--logs-dir", dest="logsdir",
help="Directory where to store logs, default is OUTPUT_DIR/logs.")
dir_group.add_argument("-R", "--render-path", dest="dest",
help="Set the path to which projects should be rendered, default is OUTPUT_DIR/rendered")
dir_group.add_argument("-p", "--medias-paths", dest="paths", action="append",
help="Paths in which to look for media files")
dir_group.add_argument("-a", "--clone-dir", dest="clone_dir",
help="Paths where to clone the testuite to run "
" default is MAIN_DIR/gst-integration-testsuites")
dir_group.add_argument("-rl", "--redirect-logs", dest="redirect_logs",
help="Redirect logs to 'stdout' or 'sdterr'.")
dir_group.add_argument("-j", "--jobs", dest="num_jobs",
help="Number of tests to execute simultaneously",
type=int)
http_server_group = parser.add_argument_group(
"Handle the HTTP server to be created")
http_server_group.add_argument(
"--http-server-port", dest="http_server_port",
help="Port on which to run the http server on localhost")
http_server_group.add_argument(
"--http-bandwith-limitation", dest="http_bandwith",
help="The artificial bandwith limitation to introduce to the local server (in Bytes/sec) (default: 1 MBps)")
http_server_group.add_argument(
"-s", "--folder-for-http-server", dest="http_server_dir",
help="Folder in which to create an http server on localhost. Default is PATHS")
http_server_group.add_argument("--http-only", dest="httponly",
action='store_true',
help="Start the http server and quit")
assets_group = parser.add_argument_group("Handle remote assets")
assets_group.add_argument(
"-u", "--update-assets-command", dest="update_assets_command",
help="Command to update assets")
assets_group.add_argument(
"--get-assets-command", dest="get_assets_command",
help="Command to get assets")
assets_group.add_argument("--remote-assets-url", dest="remote_assets_url",
help="Url to the remote assets (default:%s)" % DEFAULT_GST_QA_ASSETS_REPO)
assets_group.add_argument("-S", "--sync", dest="sync", action="store_true",
help="Synchronize asset repository")
assets_group.add_argument("--sync-all", dest="sync_all", action="store_true",
help="Synchronize asset repository,"
" including big media files")
assets_group.add_argument("--usage", action=PrintUsage,
help="Print usage documentation")
loggable.init("GST_VALIDATE_LAUNCHER_DEBUG", True, False)
tests_launcher = _TestsLauncher(libsdir)
tests_launcher.add_options(parser)
options = LauncherConfig()
parser.parse_args(namespace=options)
if not options.cleanup():
exit(1)
if options.remote_assets_url and options.sync:
if os.path.exists(options.clone_dir):
if not update_assets(options):
exit(1)
else:
if not download_assets(options):
exit(1)
if not update_assets(options):
exit(1)
tests_launcher.set_settings(options, [])
# Ensure that the scenario manager singleton is ready to be used
ScenarioManager().config = options
tests_launcher.list_tests()
if options.list_tests:
l = tests_launcher.tests
for test in l:
printc(test)
printc("\nNumber of tests: %d" % len(l), Colors.OKGREEN)
return 0
httpsrv = HTTPServer(options)
if tests_launcher.needs_http_server() or options.httponly is True:
httpsrv.start()
if options.httponly is True:
print "Running HTTP server only"
return
e = None
try:
tests_launcher.run_tests()
except Exception as e:
pass
finally:
tests_launcher.final_report()
httpsrv.stop()
if e is not None:
raise
return 0
class GESTestsManager(TestsManager):
name = "ges"
_scenarios = ScenarioManager()
def __init__(self):
super(GESTestsManager, self).__init__()
def init(self):
try:
if "--set-scenario=" in subprocess.check_output(
[GES_LAUNCH_COMMAND, "--help"]):
return True
else:
self.warning(
"Can not use ges-launch, it seems not to be compiled against"
" gst-validate")
except subprocess.CalledProcessError as e:
self.warning("Can not use ges-launch: %s" % e)
except OSError as e:
self.warning("Can not use ges-launch: %s" % e)
def add_options(self, parser):
group = parser.add_argument_group(
"GStreamer Editing Services specific option"
" and behaviours",
description="""
The GStreamer Editing Services launcher will be usable only if GES has been compiled against GstValidate
You can simply run scenarios specifying project as args. For example the following will run all available
and activated scenarios on project.xges:
$gst-validate-launcher ges /some/ges/project.xges
Available options:""")
group.add_argument("-P",
"--projects-paths",
dest="projects_paths",
default=os.path.join(utils.DEFAULT_GST_QA_ASSETS,
"ges", "ges-projects"),
help="Paths in which to look for moved medias")
group.add_argument("-r",
"--disable-recurse-paths",
dest="disable_recurse",
default=False,
action="store_true",
help="Whether to recurse into paths to find medias")
def set_settings(self, options, args, reporter):
TestsManager.set_settings(self, options, args, reporter)
self._scenarios.config = self.options
try:
os.makedirs(utils.url2path(options.dest)[0])
except OSError:
pass
def list_tests(self):
return self.tests
def register_defaults(self, project_paths=None):
projects = list()
if not self.args:
if project_paths == None:
path = self.options.projects_paths
else:
path = project_paths
for root, dirs, files in os.walk(path):
for f in files:
if not f.endswith(".xges"):
continue
projects.append(utils.path2url(os.path.join(path, root,
f)))
else:
for proj_uri in self.args:
if not utils.isuri(proj_uri):
proj_uri = utils.path2url(proj_uri)
if os.path.exists(proj_uri):
projects.append(proj_uri)
if self.options.long_limit != 0:
scenarios = [
"none", "scrub_forward_seeking", "scrub_backward_seeking"
]
else:
scenarios = [
"play_15s", "scrub_forward_seeking_full",
"scrub_backward_seeking_full"
]
for proj_uri in projects:
# First playback casses
project = XgesProjectDescriptor(proj_uri)
for scenario_name in scenarios:
scenario = self._scenarios.get_scenario(scenario_name)
if scenario is None:
continue
if scenario.get_min_media_duration() >= (
project.get_duration() / utils.GST_SECOND):
continue
classname = "ges.playback.%s.%s" % (
scenario.name, os.path.basename(proj_uri).replace(
".xges", ""))
self.add_test(
GESPlaybackTest(classname,
self.options,
self.reporter,
project,
scenario=scenario))
# And now rendering casses
for comb in GES_ENCODING_TARGET_COMBINATIONS:
classname = "ges.render.%s.%s" % (str(comb).replace(
' ', '_'), os.path.splitext(os.path.basename(proj_uri))[0])
self.add_test(
GESRenderTest(classname,
self.options,
self.reporter,
project,
combination=comb))
class PitiviTestsManager(TestsManager):
name = "pitivi"
"""The name identifying this test manager class."""
_scenarios = ScenarioManager()
def init(self):
"""Initializes the manager."""
self.fixme("Implement init checking")
return True
def add_options(self, parser):
"""Adds options to the specified ArgumentParser."""
group = parser.add_argument_group(
"Pitivi specific option group"
" and behaviours", description="")
group.add_argument("--pitivi-executable",
dest="pitivi_executable",
default=os.path.join("..", "..", "bin", "pitivi"),
help="Path to the pitivi executable")
group.add_argument("--pitivi-scenarios-paths",
dest="pitivi_scenario_paths",
help="Paths in which to look for scenario files")
def set_settings(self, options, args, reporter):
"""Configures the manager based on the specified options."""
TestsManager.set_settings(self, options, args, reporter)
PitiviTestsManager._scenarios.config = self.options
try:
os.makedirs(utils.url2path(options.dest)[0])
except OSError:
pass
def list_tests(self):
"""Lists the tests in the order they have been added."""
return self.tests
def find_scenarios(self):
"""Yields paths to the found scenario files."""
for path in self.options.pitivi_scenario_paths:
for root, unused_dirs, files in os.walk(path):
for file in files:
if not file.endswith(".scenario"):
continue
yield os.path.join(path, root, file)
def register_defaults(self):
"""Adds the available scenario files as tests."""
for scenario_name in self.find_scenarios():
scenario = PitiviTestsManager._scenarios.get_scenario(
scenario_name)
if scenario is None:
continue
classname = "pitivi.%s" % scenario.name
self.add_test(
PitiviTest(self.options.pitivi_executable,
classname,
self.options,
self.reporter,
scenario=scenario))
def main(libsdir):
if "--help" in sys.argv:
_help_message = HELP
else:
_help_message = "Use --help for the full help"
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter,
prog='gst-validate-launcher', description=_help_message)
parser.add_argument('testsuites', metavar='N', nargs='*',
help="""Lets you specify a file where the testsuite to execute is defined.
In the module if you want to work with a specific test manager(s) (for example,
'ges' or 'validate'), you should define the TEST_MANAGER variable in the
testsuite file (it can be a list of test manager names)
In this file you should implement a setup_tests function. That function takes
a TestManager and the GstValidateLauncher option as parameters and return True
if it succeeded loading the tests, False otherwise.
You will be able to configure the TestManager with its various methods. This
function will be called with each TestManager usable, for example you will be
passed the 'validate' TestManager in case the GstValidateManager launcher is
available. You should configure it using:
* test_manager.add_scenarios: which allows you to register a list of scenario names to be run
* test_manager.set_default_blacklist: Lets you set a list of tuple of the form:
(@regex_defining_blacklister_test_names, @reason_for_the_blacklisting)
* test_manager.add_generators: which allows you to register a list of #GstValidateTestsGenerator
to be used to generate tests
* test_manager.add_encoding_formats:: which allows you to register a list #MediaFormatCombination to be used for transcoding tests
You can also set default values with:
* test_manager.register_defaults: Sets default values for all parametters
* test_manager.register_default_test_generators: Sets default values for the TestsGenerators to be used
* test_manager.register_default_scenarios: Sets default values for the scenarios to be executed
* test_manager.register_default_encoding_formats: Sets default values for the encoding formats to be tested
Note that all testsuite should be inside python modules, so the directory should contain a __init__.py file
""",
default=["validate"])
parser.add_argument("-d", "--debug", dest="debug",
action="store_true",
help="Let user debug the process on timeout")
parser.add_argument("-f", "--forever", dest="forever",
action="store_true",
help="Keep running tests until one fails")
parser.add_argument("-F", "--fatal-error", dest="fatal_error",
action="store_true",
help="Stop on first fail")
parser.add_argument("--fail-on-testlist-change",
dest="fail_on_testlist_change",
action="store_true",
help="Fail the testsuite if a test has been added"
" or removed without being explicitely added/removed "
"from the testlist file.")
parser.add_argument("-t", "--wanted-tests", dest="wanted_tests",
action="append",
help="Define the tests to execute, it can be a regex."
" If it contains defaults_only, only default scenarios"
" will be executed")
parser.add_argument("-b", "--blacklisted-tests", dest="blacklisted_tests",
action="append",
help="Define the tests not to execute, it can be a regex.")
parser.add_argument("-L", "--list-tests",
dest="list_tests",
action="store_true",
help="List tests and exit")
parser.add_argument("-m", "--mute", dest="mute",
action="store_true",
help="Mute playback output, which means that we use "
"a fakesink")
parser.add_argument("-n", "--no-color", dest="no_color",
action="store_true",
help="Set it to output no colored text in the terminal")
parser.add_argument("-g", "--generate-media-info", dest="generate_info",
action="store_true",
help="Set it in order to generate the missing .media_infos files")
parser.add_argument("--update-media-info", dest="update_media_info",
action="store_true",
help="Set it in order to update exising .media_infos files")
parser.add_argument(
"-G", "--generate-media-info-with-frame-detection", dest="generate_info_full",
action="store_true",
help="Set it in order to generate the missing .media_infos files. "
"It implies --generate-media-info but enabling frame detection")
parser.add_argument("-lt", "--long-test-limit", dest="long_limit",
action='store',
help="Defines the limit for which a test is considered as long (in seconds)."
" Note that 0 will enable all tests", type=int),
parser.add_argument("--dump-on-failure", dest="dump_on_failure",
action="store_true", default=False,
help="Dump logs to stdout when a test fails")
parser.add_argument("-c", "--config", dest="config",
help="This is DEPRECATED, prefer using the testsuite format"
" to configure testsuites")
parser.add_argument("-vg", "--valgrind", dest="valgrind",
action="store_true",
help="Run the tests inside Valgrind")
parser.add_argument("--gdb", dest="gdb",
action="store_true",
help="Run the tests inside gdb (implies"
" --output-dir=stdout and --jobs=1)")
parser.add_argument("-nd", "--no-display", dest="no_display",
action="store_true",
help="Run the tests without outputting graphics"
" on any display. It tries to run all graphical operation"
" in a virtual framebuffer."
" Note that it is currently implemented only"
" for the X server thanks to Xvfb (which is requeried in that case)")
dir_group = parser.add_argument_group(
"Directories and files to be used by the launcher")
parser.add_argument('--xunit-file', action='store',
dest='xunit_file', metavar="FILE",
help=("Path to xml file to store the xunit report in."))
dir_group.add_argument("-M", "--main-dir", dest="main_dir",
help="Main directory where to put files. Default is %s" % DEFAULT_MAIN_DIR)
dir_group.add_argument("--testsuites-dir", dest="testsuites_dir",
help="Directory where to look for testsuites. Default is %s"
" Note that GstValidate expect testsuite file to have .testsuite"
" as an extension in this folder." % DEFAULT_TESTSUITES_DIR)
dir_group.add_argument("-o", "--output-dir", dest="output_dir",
help="Directory where to store logs and rendered files. Default is MAIN_DIR")
dir_group.add_argument("-l", "--logs-dir", dest="logsdir",
help="Directory where to store logs, default is OUTPUT_DIR/logs.")
dir_group.add_argument("-R", "--render-path", dest="dest",
help="Set the path to which projects should be rendered, default is OUTPUT_DIR/rendered")
dir_group.add_argument("-p", "--medias-paths", dest="user_paths", action="append",
help="Paths in which to look for media files")
dir_group.add_argument("-a", "--clone-dir", dest="clone_dir",
help="Paths where to clone the testuite to run "
" default is MAIN_DIR/gst-integration-testsuites")
dir_group.add_argument("-rl", "--redirect-logs", dest="redirect_logs",
help="Redirect logs to 'stdout' or 'sdterr'.")
dir_group.add_argument("-j", "--jobs", dest="num_jobs",
help="Number of tests to execute simultaneously",
type=int)
http_server_group = parser.add_argument_group(
"Handle the HTTP server to be created")
http_server_group.add_argument(
"--http-server-port", dest="http_server_port",
help="Port on which to run the http server on localhost", type=int)
http_server_group.add_argument(
"--http-bandwith-limitation", dest="http_bandwith",
help="The artificial bandwith limitation to introduce to the local server (in Bytes/sec) (default: 1 MBps)")
http_server_group.add_argument(
"-s", "--folder-for-http-server", dest="http_server_dir",
help="Folder in which to create an http server on localhost. Default is PATHS")
http_server_group.add_argument("--http-only", dest="httponly",
action='store_true',
help="Start the http server and quit")
assets_group = parser.add_argument_group("Handle remote assets")
assets_group.add_argument(
"--get-assets-command", dest="get_assets_command",
help="Command to get assets")
assets_group.add_argument("--remote-assets-url", dest="remote_assets_url",
help="Url to the remote assets (default:%s)" % DEFAULT_GST_QA_ASSETS_REPO)
assets_group.add_argument("-S", "--sync", dest="sync", action="store_true",
help="Synchronize asset repository")
assets_group.add_argument("-fs", "--force-sync", dest="force_sync", action="store_true",
help="Synchronize asset repository reseting any change that might have"
" happened in the testsuite")
assets_group.add_argument("--sync-all", dest="sync_all", action="store_true",
help="Synchronize asset repository,"
" including big media files")
assets_group.add_argument("--usage", action=PrintUsage,
help="Print usage documentation")
loggable.init("GST_VALIDATE_LAUNCHER_DEBUG", True, False)
tests_launcher = _TestsLauncher(libsdir)
tests_launcher.add_options(parser)
if _help_message == HELP and which(LESS):
tmpf = tempfile.NamedTemporaryFile()
parser.print_help(file=tmpf)
exit(os.system("%s %s" % (LESS, tmpf.name)))
options = LauncherConfig()
parser.parse_args(namespace=options)
if not options.cleanup():
exit(1)
if options.remote_assets_url and options.sync and not os.path.exists(options.clone_dir):
if not download_assets(options):
exit(1)
# Ensure that the scenario manager singleton is ready to be used
ScenarioManager().config = options
if not tests_launcher.set_settings(options, []):
exit(1)
if tests_launcher.list_tests() == -1:
printc("\nFailling as tests have been removed/added "
" (--fail-on-testlist-change)", Colors.FAIL)
exit(1)
if options.list_tests:
l = tests_launcher.tests
for test in l:
printc(test)
printc("\nNumber of tests: %d" % len(l), Colors.OKGREEN)
return 0
httpsrv = HTTPServer(options)
if tests_launcher.needs_http_server() or options.httponly is True:
httpsrv.start()
vfb_server = get_virual_frame_buffer_server(options)
if options.no_display:
res = vfb_server.start()
if res[0] is False:
printc("Could not start virtual frame server: %s" % res[1],
Colors.FAIL)
exit(1)
os.environ["DISPLAY"] = vfb_server.display_id
if options.httponly is True:
print("Running HTTP server only")
return
# There seems to be some issue with forking, dconf and some gtype
# initialization that deadlocks occasionally, setting the
# GSettings backend make it go away.
# Also happened here: https://cgit.freedesktop.org/gstreamer/gst-plugins-good/commit/tests/check/Makefile.am?id=8e2c1d1de56bddbff22170f8b17473882e0e63f9
os.environ['GSETTINGS_BACKEND'] = "memory"
e = None
try:
tests_launcher.run_tests()
except Exception as e:
pass
finally:
tests_launcher.final_report()
tests_launcher.clean_tests()
httpsrv.stop()
vfb_server.stop()
if e is not None:
raise
return 0