def _FakeArgManager(): return arg_manager.AndroidArgsManager( catalog_manager.AndroidCatalogManager(fake_args.AndroidCatalog()), fake_args.TypedArgRules(), fake_args.SharedArgRules())
def Run(self, args): """Run the 'gcloud firebase test run' command to invoke a test in Test Lab. Args: args: an argparse namespace. All the arguments that were provided to this command invocation (i.e. group and command arguments combined). Returns: One of: - a list of TestOutcome tuples (if ToolResults are available). - a URL string pointing to the user's results in ToolResults or GCS. """ if args.async_ and not args.IsSpecified('format'): args.format = """ value(format( 'Final test results will be available at [{0}].', []) ) """ log.status.Print( '\nHave questions, feedback, or issues? Get support by ' 'visiting:\n https://firebase.google.com/support/\n') arg_manager.AndroidArgsManager().Prepare(args) project = util.GetProject() tr_client = self.context['toolresults_client'] tr_messages = self.context['toolresults_messages'] storage_client = self.context['storage_client'] bucket_ops = results_bucket.ResultsBucketOps(project, args.results_bucket, args.results_dir, tr_client, tr_messages, storage_client) bucket_ops.UploadFileToGcs(args.app, _APK_MIME_TYPE) if args.test: bucket_ops.UploadFileToGcs(args.test, _APK_MIME_TYPE) for obb_file in (args.obb_files or []): bucket_ops.UploadFileToGcs(obb_file, 'application/octet-stream') if getattr(args, 'robo_script', None): bucket_ops.UploadFileToGcs(args.robo_script, 'application/json') additional_apks = getattr(args, 'additional_apks', None) or [] for additional_apk in additional_apks: bucket_ops.UploadFileToGcs(additional_apk, _APK_MIME_TYPE) other_files = getattr(args, 'other_files', None) or {} for device_path, file_to_upload in six.iteritems(other_files): bucket_ops.UploadFileToGcs( file_to_upload, None, destination_object=util.GetRelativeDevicePath(device_path)) bucket_ops.LogGcsResultsUrl() tr_history_picker = history_picker.ToolResultsHistoryPicker( project, tr_client, tr_messages) history_name = PickHistoryName(args) history_id = tr_history_picker.GetToolResultsHistoryId(history_name) matrix = matrix_creator.CreateMatrix( args, self.context, history_id, bucket_ops.gcs_results_root, six.text_type(self.ReleaseTrack())) monitor = matrix_ops.MatrixMonitor(matrix.testMatrixId, args.type, self.context) with ctrl_c_handler.CancellableTestSection(monitor): supported_executions = monitor.HandleUnsupportedExecutions(matrix) tr_ids = tool_results.GetToolResultsIds(matrix, monitor) url = tool_results.CreateToolResultsUiUrl(project, tr_ids) log.status.Print('') if args.async_: return url log.status.Print( 'Test results will be streamed to [{0}].'.format(url)) # If we have exactly one testExecution, show detailed progress info. if len(supported_executions ) == 1 and args.num_flaky_test_attempts == 0: monitor.MonitorTestExecutionProgress( supported_executions[0].id) else: monitor.MonitorTestMatrixProgress() log.status.Print('\nMore details are available at [{0}].'.format(url)) # Fetch the per-dimension test outcomes list, and also the "rolled-up" # matrix outcome from the Tool Results service. summary_fetcher = results_summary.ToolResultsSummaryFetcher( project, tr_client, tr_messages, tr_ids) self.exit_code = exit_code.ExitCodeFromRollupOutcome( summary_fetcher.FetchMatrixRollupOutcome(), tr_messages.Outcome.SummaryValueValuesEnum) return summary_fetcher.CreateMatrixOutcomeSummaryUsingEnvironments()
class _BaseRun(object): """Invoke a test in Firebase Test Lab for Android and view test results.""" detailed_help = { 'DESCRIPTION': """\ *{command}* invokes and monitors tests in Firebase Test Lab for Android. Three main types of Android tests are currently supported: - *robo*: runs a smart, automated exploration of the activities in your Android app which records any installation failures or crashes and builds an activity map with associated screenshots and video. - *instrumentation*: runs automated unit or integration tests written using a testing framework. Firebase Test Lab for Android currently supports the Espresso, Robotium and UI Automator 2.0 testing frameworks. - *game-loop*: executes a special intent built into the game app (a "demo mode") that simulates the actions of a real player. This test type can include multiple game loops (also called "scenarios"), which can be logically organized using scenario labels so that you can run related loops together. Refer to https://firebase.google.com/docs/test-lab/android/game-loop for more information about how to build and run Game Loop tests. The type of test to run can be specified with the *--type* flag, although the type can often be inferred from other flags. Specifically, if the *--test* flag is present, the test *--type* defaults to `instrumentation`. If *--test* is not present, then *--type* defaults to `robo`. All arguments for *{command}* may be specified on the command line and/or within an argument file. Run *$ gcloud topic arg-files* for more information about argument files. """, 'EXAMPLES': """\ To invoke a robo test lasting 100 seconds against the default device environment, run: $ {command} --app APP_APK --timeout 100s When specifying devices to test against, the preferred method is to use the --device flag. For example, to invoke a robo test against a virtual, generic MDPI Nexus device in landscape orientation, run: $ {command} --app APP_APK --device model=NexusLowRes,orientation=landscape To invoke an instrumentation test against a physical Nexus 6 device (MODEL_ID: shamu) which is running Android API level 21 in French, run: $ {command} --app APP_APK --test TEST_APK --device model=shamu,version=21,locale=fr To test against multiple devices, specify --device more than once: $ {command} --app APP_APK --test TEST_APK --device model=Nexus4,version=19 --device model=Nexus4,version=21 --device model=NexusLowRes,version=25 You may also use the legacy dimension flags (deprecated) to specify which devices to use. Firebase Test Lab will run tests against every possible combination of the listed device dimensions. Note that some combinations of device models and OS versions may not be valid or available in Test Lab. Any unsupported combinations of dimensions in the test matrix will be skipped. For example, to execute a series of 5-minute robo tests against a very comprehensive matrix of virtual and physical devices, OS versions, locales and orientations, run: $ {command} --app APP_APK --timeout 5m --device-ids=shamu,NexusLowRes,Nexus5,g3,zeroflte --os-version-ids=19,21,22,23,24,25 --locales=en_GB,es,fr,ru,zh --orientations=portrait,landscape The above command will generate a test matrix with a total of 300 test executions, but only the subset of executions with valid dimension combinations will actually run your tests. Controlling Results Storage By default, Firebase Test Lab stores detailed test results for a limited time in a Google Cloud Storage bucket provided for you at no charge. If you wish to use a storage bucket that you control, or if you need to retain detailed test results for a longer period, use the *--results-bucket* option. See https://firebase.google.com/docs/test-lab/analyzing-results#detailed for more information. Detailed test result files are prefixed by default with a timestamp and a random character string. If you require a predictable path where detailed test results are stored within the results bucket (say, if you have a Continuous Integration system which does custom post-processing of test result artifacts), use the *--results-dir* option. _Note that each test invocation *must* have a unique storage location, so never reuse the same value for *--results-dir* between different test runs_. Possible strategies could include using a UUID or sequence number for *--results-dir*. For example, to run a robo test using a specific Google Cloud Storage location to hold the raw test results, run: $ {command} --app APP_APK --results-bucket=gs://my-bucket --results-dir=my/test/results/<unique-value> To run an instrumentation test and specify a custom name under which the history of your tests will be collected and displayed in the Firebase console, run: $ {command} --app APP_APK --test TEST_APK --results-history-name='Excelsior App Test History' Argument Files All test arguments for a given test may alternatively be stored in an argument group within a YAML-formatted argument file. The _ARG_FILE_ may contain one or more named argument groups, and argument groups may be combined using the `include:` attribute (Run *$ gcloud topic arg-files* for more information). The ARG_FILE can easily be shared with colleagues or placed under source control to ensure consistent test executions. To run a test using arguments loaded from an ARG_FILE named *excelsior_args*, which contains an argument group named *robo-args:*, use the following syntax: $ {command} path/to/excelsior_args:robo-args """, } def Run(self, args): """Run the 'gcloud firebase test run' command to invoke a test in Test Lab. Args: args: an argparse namespace. All the arguments that were provided to this command invocation (i.e. group and command arguments combined). Returns: One of: - a list of TestOutcome tuples (if ToolResults are available). - a URL string pointing to the user's results in ToolResults or GCS. """ if args. async and not args.IsSpecified('format'): args.format = """ value(format( 'Final test results will be available at [{0}].', []) ) """ log.status.Print( '\nHave questions, feedback, or issues? Get support by ' 'visiting:\n https://firebase.google.com/support/\n') arg_manager.AndroidArgsManager().Prepare(args) project = util.GetProject() tr_client = self.context['toolresults_client'] tr_messages = self.context['toolresults_messages'] storage_client = self.context['storage_client'] bucket_ops = results_bucket.ResultsBucketOps(project, args.results_bucket, args.results_dir, tr_client, tr_messages, storage_client) bucket_ops.UploadFileToGcs(args.app) if args.test: bucket_ops.UploadFileToGcs(args.test) for obb_file in (args.obb_files or []): bucket_ops.UploadFileToGcs(obb_file) if getattr(args, 'robo_script', None): bucket_ops.UploadFileToGcs(args.robo_script) additional_apks = getattr(args, 'additional_apks', None) or [] for additional_apk in additional_apks: bucket_ops.UploadFileToGcs(additional_apk) for other_files in getattr(args, 'other-files', None) or {}: bucket_ops.UploadFileToGcs(other_files) bucket_ops.LogGcsResultsUrl() tr_history_picker = history_picker.ToolResultsHistoryPicker( project, tr_client, tr_messages) history_name = PickHistoryName(args) history_id = tr_history_picker.GetToolResultsHistoryId(history_name) matrix = matrix_creator.CreateMatrix(args, self.context, history_id, bucket_ops.gcs_results_root, str(self.ReleaseTrack())) monitor = matrix_ops.MatrixMonitor(matrix.testMatrixId, args.type, self.context) with ctrl_c_handler.CancellableTestSection(monitor): supported_executions = monitor.HandleUnsupportedExecutions(matrix) tr_ids = tool_results.GetToolResultsIds(matrix, monitor) url = tool_results.CreateToolResultsUiUrl(project, tr_ids) log.status.Print('') if args. async: return url log.status.Print( 'Test results will be streamed to [{0}].'.format(url)) # If we have exactly one testExecution, show detailed progress info. if len(supported_executions) == 1: monitor.MonitorTestExecutionProgress( supported_executions[0].id) else: monitor.MonitorTestMatrixProgress() log.status.Print('\nMore details are available at [{0}].'.format(url)) # Fetch the per-dimension test outcomes list, and also the "rolled-up" # matrix outcome from the Tool Results service. summary_fetcher = results_summary.ToolResultsSummaryFetcher( project, tr_client, tr_messages, tr_ids) self.exit_code = exit_code.ExitCodeFromRollupOutcome( summary_fetcher.FetchMatrixRollupOutcome(), tr_messages.Outcome.SummaryValueValuesEnum) return summary_fetcher.CreateMatrixOutcomeSummary()
def PrepareAndroidArgs(args): cat_mgr = catalog_manager.AndroidCatalogManager(fake_args.AndroidCatalog()) arg_manager.AndroidArgsManager(cat_mgr).Prepare(args)
def _AndroidArgManagerWithFakeCatalog(): cat_mgr = catalog_manager.AndroidCatalogManager(fake_args.AndroidCatalog()) return arg_manager.AndroidArgsManager(cat_mgr)
def Run(self, args): """Run the 'gcloud firebase test run' command to invoke a test in Test Lab. Args: args: an argparse namespace. All the arguments that were provided to this command invocation (i.e. group and command arguments combined). Returns: One of: - a list of TestOutcome tuples (if ToolResults are available). - a URL string pointing to the user's results in ToolResults or GCS. """ arg_manager.AndroidArgsManager().Prepare(args) project = util.GetProject() tr_client = self.context['toolresults_client'] tr_messages = self.context['toolresults_messages'] storage_client = self.context['storage_client'] bucket_ops = results_bucket.ResultsBucketOps(project, args.results_bucket, args.results_dir, tr_client, tr_messages, storage_client) bucket_ops.UploadFileToGcs(args.app) if args.test: bucket_ops.UploadFileToGcs(args.test) for obb_file in (args.obb_files or []): bucket_ops.UploadFileToGcs(obb_file) bucket_ops.LogGcsResultsUrl() tr_history_picker = history_picker.ToolResultsHistoryPicker( project, tr_client, tr_messages) history_name = PickHistoryName(args) history_id = tr_history_picker.GetToolResultsHistoryId(history_name) matrix = matrix_creator.CreateMatrix(args, self.context, history_id, bucket_ops.gcs_results_root) monitor = matrix_ops.MatrixMonitor(matrix.testMatrixId, args.type, self.context) with ctrl_c_handler.CancellableTestSection(monitor): supported_executions = monitor.HandleUnsupportedExecutions(matrix) tr_ids = tool_results.GetToolResultsIds(matrix, monitor) url = tool_results.CreateToolResultsUiUrl(project, tr_ids) log.status.Print('') if args. async: return url log.status.Print( 'Test results will be streamed to [{0}].'.format(url)) # If we have exactly one testExecution, show detailed progress info. if len(supported_executions) == 1: monitor.MonitorTestExecutionProgress( supported_executions[0].id) else: monitor.MonitorTestMatrixProgress() log.status.Print('\nMore details are available at [{0}].'.format(url)) # Fetch the per-dimension test outcomes list, and also the "rolled-up" # matrix outcome from the Tool Results service. summary_fetcher = results_summary.ToolResultsSummaryFetcher( project, tr_client, tr_messages, tr_ids) self.exit_code = exit_code.ExitCodeFromRollupOutcome( summary_fetcher.FetchMatrixRollupOutcome(), tr_messages.Outcome.SummaryValueValuesEnum) return summary_fetcher.CreateMatrixOutcomeSummary()