def testDirectoriesToPull_GoodInput(self):
args = arg_file.GetArgsFromArgFile(
GOOD_ARGS + ':directories-to-pull.list', self.android_args_set)
self.assertEqual(args['directories_to_pull'],
['/sdcard/tempDirUno', '/sdcard/tempDirDos'])
args = arg_file.GetArgsFromArgFile(
GOOD_ARGS + ':directories-to-pull.string', self.android_args_set)
self.assertEqual(args['directories_to_pull'], ['/singleStringPath'])
def testGoodArgFile_FileArgUsedIfCliArgIsNone(self):
args = test_utils.NewNameSpace([], test='jill.apk', results_bucket=None)
file_args = arg_file.GetArgsFromArgFile(GOOD_ARGS + ':test-override',
COMMON_ARGS)
arg_util.ApplyLowerPriorityArgs(args, file_args, True)
self.assertEqual(args.results_bucket, 'gs://arg-file-bucket')
self.AssertErrContains('')
def testInclude_FormsAGraphCycle(self):
with self.assertRaises(calliope_exceptions.InvalidArgumentException) as e:
arg_file.GetArgsFromArgFile(INCLUDES + ':incl-cycle-a',
self.android_args_set)
self.assertIn(
'[include]: Detected cyclic reference to arg group [incl-cycle-a]',
six.text_type(e.exception))
def testGrantPermissions_InvalidValueUsed(self):
with self.assertRaises(calliope_exceptions.InvalidArgumentException) as e:
arg_file.GetArgsFromArgFile(BAD_ARGS + ':grant-permissions.bad',
self.android_args_set)
self.assertIn('[grant-permissions]: Invalid permissions',
six.text_type(e.exception))
def testScenarioLabels_InvalidNestedListValue(self):
with self.assertRaises(calliope_exceptions.InvalidArgumentException) as e:
arg_file.GetArgsFromArgFile(STR_LISTS + ':labels-nested-list',
self.android_args_set)
msg = six.text_type(e.exception)
self.assertIn('Invalid value for [scenario-labels]:', msg)
self.assertIn("['label2', 'label3']", msg)
def testInclude_SimpleCase(self):
args = arg_file.GetArgsFromArgFile(INCLUDES + ':group3',
self.android_args_set)
self.assertEqual(args['async_'], True) # Value from includer
self.assertEqual(args['type'], 'instrumentation') # Value from includee
self.assertListEqual(args['locales'], ['de', 'en', 'it']) # From includer
self.assertListEqual(args['os_version_ids'], ['21']) # From includee
def testScenarioLabels_InvalidDictValue(self):
with self.assertRaises(calliope_exceptions.InvalidArgumentException) as e:
arg_file.GetArgsFromArgFile(STR_LISTS + ':labels-dict',
self.android_args_set)
msg = six.text_type(e.exception)
self.assertIn('Invalid value for [scenario-labels]: {', msg)
self.assertIn("'label1': 'foo'", msg)
def testOrientationList_DuplicateCase2(self):
with self.assertRaises(calliope_exceptions.InvalidArgumentException) as e:
arg_file.GetArgsFromArgFile(STR_LISTS + ':ori-dup2',
self.android_args_set)
self.assertEqual(
'Invalid value for [orientations]: orientations may not be repeated.',
six.text_type(e.exception))
def testIntList_InvalidDictValue(self):
with self.assertRaises(calliope_exceptions.InvalidArgumentException) as e:
arg_file.GetArgsFromArgFile(INTEGERS + ':scenario-dict',
self.android_args_set)
msg = six.text_type(e.exception)
self.assertIn('Invalid value for [scenario-numbers]: {', msg)
self.assertIn("'foo'", msg)
def testIntList_InvalidListOfStrings(self):
with self.assertRaises(
calliope_exceptions.InvalidArgumentException) as e:
arg_file.GetArgsFromArgFile(INTEGERS + ':scenario-str-list',
self.android_args_set)
msg = six.text_type(e.exception)
self.assertIn('Invalid value for [scenario-numbers]: 1', msg)
def testTestTargetsForShard_GoodInput(self):
args = arg_file.GetArgsFromArgFile(
GOOD_ARGS + ':test-targets-for-shard.good', self.android_args_set)
self.assertEqual(args['test_targets_for_shard'], [
'class com.ClassForShard1#flakyTest1;class com.ClassForShard1#flakyTest2',
'class com.ClassForShard2#flakyTest3,com.ClassForShard2#flakyTest4'
])
def testIntList_ZeroIsNotAPositiveInt(self):
with self.assertRaises(calliope_exceptions.InvalidArgumentException) as e:
arg_file.GetArgsFromArgFile(INTEGERS + ':scenario-zero',
self.android_args_set)
msg = six.text_type(e.exception)
self.assertIn('Invalid value for [scenario-numbers]:', msg)
self.assertIn('Value must be greater than or equal to 1; received: 0', msg)
def testEnvironmentVariables_GoodInput(self):
args = arg_file.GetArgsFromArgFile(
GOOD_ARGS + ':environment-variables.good', self.android_args_set)
self.assertEqual(args['environment_variables'], {
'e1': 'value1',
'e2': 'value2'
})
def testInclude_GroupIsNestedList(self):
with self.assertRaises(
calliope_exceptions.InvalidArgumentException) as e:
arg_file.GetArgsFromArgFile(INCLUDES + ':incl-nested',
self.android_args_set)
self.assertEqual("Invalid value for [include]: ['group3', 'group4']",
six.text_type(e.exception))
def testDevice_ValidSparseMatrix_TerseSyntax1(self):
args = arg_file.GetArgsFromArgFile(GOOD_ARGS + ':sparse-terse1',
self.android_args_set)
self.assertEqual(len(args['device']), 3)
self.assertDictEqual(args['device'][0], {'model': 'Nexus5'})
self.assertDictEqual(args['device'][1], {'model': 'sailfish'})
self.assertDictEqual(args['device'][2], {'model': 'shamu'})
def testInclude_GroupNameIsMissing(self):
with self.assertRaises(calliope_exceptions.BadFileException) as e:
arg_file.GetArgsFromArgFile(INCLUDES + ':incl-missing',
self.android_args_set)
self.assertIn(
'Could not find argument group [missing] in argument file.',
six.text_type(e.exception))
def Prepare(self, args):
"""Load, apply defaults, and perform validation on test arguments.
Args:
args: an argparse namespace. All the arguments that were provided to this
gcloud command invocation (i.e. group and command arguments combined).
Arg values from an optional arg-file and/or arg default values may be
added to this argparse namespace.
Raises:
InvalidArgumentException: If an argument name is unknown, an argument does
not contain a valid value, or an argument is not valid when used with
the given type of test.
RequiredArgumentException: If a required arg is missing.
"""
all_test_args_set = arg_util.GetSetOfAllTestArgs(self._typed_arg_rules,
self._shared_arg_rules)
args_from_file = arg_file.GetArgsFromArgFile(args.argspec,
all_test_args_set)
arg_util.ApplyLowerPriorityArgs(args, args_from_file, True)
test_type = self.GetTestTypeOrRaise(args)
typed_arg_defaults = self._typed_arg_rules[test_type]['defaults']
shared_arg_defaults = self._shared_arg_rules['defaults']
arg_util.ApplyLowerPriorityArgs(args, typed_arg_defaults)
arg_util.ApplyLowerPriorityArgs(args, shared_arg_defaults)
arg_validate.ValidateArgsForTestType(args, test_type, self._typed_arg_rules,
self._shared_arg_rules,
all_test_args_set)
arg_validate.ValidateIosDeviceList(args, self._catalog_mgr)
arg_validate.ValidateResultsBucket(args)
arg_validate.ValidateResultsDir(args)
def testIntList_InvalidNegativeIntInList(self):
with self.assertRaises(calliope_exceptions.InvalidArgumentException) as e:
arg_file.GetArgsFromArgFile(INTEGERS + ':scenario-neg-in-list',
self.android_args_set)
msg = six.text_type(e.exception)
self.assertIn('Invalid value for [scenario-numbers]:', msg)
self.assertIn('must be greater than or equal to 1; received: -1', msg)
def testDevice_InvalidSparseMatrix_2(self):
with self.assertRaises(
calliope_exceptions.InvalidArgumentException) as e:
arg_file.GetArgsFromArgFile(BAD_ARGS + ':sparse2',
self.android_args_set)
self.assertIn('Invalid value for [device]:',
six.text_type(e.exception))
def testGoodArgFile_FileArgDoesNotOverrideCliArg(self):
args = test_utils.NewNameSpace([], test='jill.apk', results_bucket='jack')
file_args = arg_file.GetArgsFromArgFile(GOOD_ARGS + ':test-override',
COMMON_ARGS)
arg_util.ApplyLowerPriorityArgs(args, file_args, True)
self.assertEqual(args.results_bucket, 'jack')
self.AssertErrContains('"--results-bucket jack" overrides file argument ')
self.AssertErrContains('"results-bucket: gs://arg-file-bucket"')
def testOtherFiles_GoodInput(self):
args = arg_file.GetArgsFromArgFile(GOOD_ARGS + ':other-files.good',
self.android_args_set)
self.assertEqual(
args['other_files'], {
'/sdcard/dir1/file.txt': 'local/dir/file1.txt',
'/sdcard/dir2/file.txt': 'gs://bucket/file2.txt'
})
def testDeviceIdList_NestedList(self):
with self.assertRaises(
calliope_exceptions.InvalidArgumentException) as e:
arg_file.GetArgsFromArgFile(STR_LISTS + ':dev-nestedlist',
self.android_args_set)
self.assertEqual(
"Invalid value for [device-ids]: ['Nexus6', 'Nexus7']",
six.text_type(e.exception))
def testInclude_MoreComplexCase(self):
args = arg_file.GetArgsFromArgFile(INCLUDES + ':top-group',
self.android_args_set)
# Check that topmost group value with no matching included values is used
self.assertEqual(args['app'], 'path/to/peekaboo.apk')
# Check that topmost group value not overridden by directly included value
self.assertEqual(args['timeout'], 600)
# Check that topmost group value not overridden by indirectly included value
self.assertEqual(args['type'], 'robo')
# Check that included groups in a list are processed left-to-right
self.assertEqual(args['async_'], False)
# Check that 2-levels-deep includee doesn't override 1-level-deep includee
self.assertListEqual(args['locales'], ['de', 'en', 'it'])
# Check that value from 2-levels-deep includee is used if no overrides
self.assertListEqual(args['os_version_ids'], ['21'])
def testDevice_ValidSparseMatrix_TerseSyntax2(self):
args = arg_file.GetArgsFromArgFile(GOOD_ARGS + ':sparse-terse2',
self.android_args_set)
self.assertEqual(len(args['device']), 3)
d1 = args['device'][0]
d2 = args['device'][1]
d3 = args['device'][2]
self.assertDictEqual(
d1, {
'model': 'NexusLowRes',
'version': '23',
'locale': 'es',
'orientation': 'landscape'
})
self.assertDictEqual(d2, {'model': 'shamu'})
self.assertDictEqual(d3, {'version': '5.1.x', 'locale': 'zh'})
def Prepare(self, args):
"""Load, apply defaults, and perform validation on test arguments.
Args:
args: an argparse namespace. All the arguments that were provided to this
gcloud command invocation (i.e. group and command arguments combined).
Arg values from an optional arg-file and/or arg default values may be
added to this argparse namespace.
Raises:
InvalidArgumentException: If an argument name is unknown, an argument does
not contain a valid value, or an argument is not valid when used with
the given type of test.
RequiredArgumentException: If a required arg is missing.
"""
all_test_args_set = arg_util.GetSetOfAllTestArgs(
self._typed_arg_rules, self._shared_arg_rules)
args_from_file = arg_file.GetArgsFromArgFile(args.argspec,
all_test_args_set)
arg_util.ApplyLowerPriorityArgs(args, args_from_file, True)
test_type = self.GetTestTypeOrRaise(args)
self._CheckForConflictingArgs(args)
typed_arg_defaults = self._typed_arg_rules[test_type]['defaults']
shared_arg_defaults = self._shared_arg_rules['defaults']
arg_util.ApplyLowerPriorityArgs(args, typed_arg_defaults)
arg_util.ApplyLowerPriorityArgs(args, shared_arg_defaults)
self._ApplyLegacyMatrixDimensionDefaults(args)
arg_validate.ValidateArgsForTestType(args, test_type,
self._typed_arg_rules,
self._shared_arg_rules,
all_test_args_set)
arg_validate.ValidateOsVersions(args, self._catalog_mgr)
arg_validate.ValidateDeviceList(args, self._catalog_mgr)
arg_validate.ValidateResultsBucket(args)
arg_validate.ValidateResultsDir(args)
arg_validate.NormalizeAndValidateObbFileNames(args.obb_files)
arg_validate.ValidateRoboDirectivesList(args)
arg_validate.ValidateEnvironmentVariablesList(args)
arg_validate.ValidateTestTargetsForShard(args)
arg_validate.NormalizeAndValidateDirectoriesToPullList(
args.directories_to_pull)
arg_validate.ValidateScenarioNumbers(args)
def testDevice_ValidSparseMatrix_VerboseSyntax(self):
args = arg_file.GetArgsFromArgFile(GOOD_ARGS + ':sparse-verbose',
self.android_args_set)
self.assertEqual(len(args['device']), 2)
d1 = args['device'][0]
d2 = args['device'][1]
self.assertDictEqual(d1, {
'model': 'm1',
'version': 'v1',
'locale': 'l1',
'orientation': 'o1'
})
self.assertDictEqual(d2, {
'model': 'm2',
'version': 'v2',
'locale': 'l2',
'orientation': 'o2'
})
def testGoodArgFile_FileArgsMergedWithCliArgs(self):
args = self.NewTestArgs(app='jill.apk', results_bucket='jack')
file_args = arg_file.GetArgsFromArgFile(GOOD_ARGS + ':my-group',
self.android_args_set)
arg_util.ApplyLowerPriorityArgs(args, file_args, True)
self.AssertErrContains('')
# Simulated args from CLI
self.assertEqual(args.app, 'jill.apk')
self.assertEqual(args.results_bucket, 'jack')
# Args merged from GOOD_ARGS file
self.assertEqual(args.type, 'instrumentation')
self.assertEqual(args.test, 'startrek.apk')
self.assertEquals(sorted(args.locales), sorted(['klingon', 'romulan']))
self.assertEqual(args.results_dir, 'my/results/dir')
self.assertEqual(args.robo_directives, {
'resource1': 'value1',
'text:resource2': '2',
'click:resource3': ''
})
self.assertEqual(args.test_targets,
['package kirk', 'class enterprise.spock'])
def testNumUniformShards_GoodInput(self):
args = arg_file.GetArgsFromArgFile(
GOOD_ARGS + ':num-uniform-shards.good', self.android_args_set)
self.assertEqual(args['num_uniform_shards'], 2)
def testNumUniformShards_StringInput(self):
with self.assertRaises(calliope_exceptions.InvalidArgumentException):
arg_file.GetArgsFromArgFile(
BAD_ARGS + ':num-uniform-shards.string', self.android_args_set)
def testDevice_ValidSparseMatrix_EmptyDevice(self):
args = arg_file.GetArgsFromArgFile(GOOD_ARGS + ':sparse-empty',
self.android_args_set)
self.assertEqual(len(args['device']), 1)
d1 = args['device'][0]
self.assertDictEqual(d1, {})
