def _GetUploadUrls(self, filename, builder_run=None): """Returns a list of all urls for which to upload filename to. Args: filename: The filename of the file we want to upload. builder_run: builder_run object from which to get the board, base upload url, and bot_id. If none, this stage's values. """ board = None urls = [self.upload_url] bot_id = self._bot_id if builder_run: urls = [builder_run.GetArchive().upload_url] bot_id = builder_run.GetArchive().bot_id if (builder_run.config['boards'] and len(builder_run.config['boards']) == 1): board = builder_run.config['boards'][0] if (not self._IsInUploadBlacklist(filename) and (hasattr(self, '_current_board') or board)): if self._run.config.pre_cq: # Do not load artifacts.json for pre-cq configs. This is a # workaround for crbug.com/440167. return urls board = board or self._current_board custom_artifacts_file = portage_util.ReadOverlayFile( 'scripts/artifacts.json', board=board) if custom_artifacts_file is not None: json_file = json.loads(custom_artifacts_file) for url in json_file.get('extra_upload_urls', []): urls.append('/'.join([url, bot_id, self.version])) return urls
def _GetUploadUrls(self, filename, builder_run=None): """Returns a list of all urls for which to upload filename to. Args: filename: The filename of the file we want to upload. builder_run: builder_run object from which to get the board, base upload url, and bot_id. If none, this stage's values. """ board = None urls = [self.upload_url] bot_id = self._bot_id if builder_run: urls = [builder_run.GetArchive().upload_url] bot_id = builder_run.GetArchive().bot_id if (builder_run.config['boards'] and len(builder_run.config['boards']) == 1): board = builder_run.config['boards'][0] if (not self._IsInUploadBlacklist(filename) and (hasattr(self, '_current_board') or board)): board = board or self._current_board custom_artifacts_file = portage_util.ReadOverlayFile( 'scripts/artifacts.json', board=board) if custom_artifacts_file is not None: json_file = json.loads(custom_artifacts_file) for url in json_file.get('extra_upload_urls', []): # Moblab users do not need the paladin, pfq or trybot # builds, filter those bots from extra uploads. if self._FilterBuildFromMoblab(url, bot_id): continue urls.append('/'.join([url, bot_id, self.version])) return urls
def testReadOverlayFileOrder(self): """Verify that the boards are examined in the right order.""" m = self.PatchObject(os.path, 'isfile', return_value=False) portage_util.ReadOverlayFile('test', self.PUBLIC, self.PUB_PRIV, self.tempdir) read_overlays = [x[0][0][:-5] for x in m.call_args_list] overlays = [x for x in reversed(self.overlays[self.PUB_PRIV][self.PUBLIC])] self.assertEqual(read_overlays, overlays)
def GetDummyArchiveUrls(self): """Returns upload URLs for dummy artifacts based on artifacts.json.""" upload_urls = [self.dummy_archive_url] artifacts_file = portage_util.ReadOverlayFile( 'scripts/artifacts.json', board=self._current_board, buildroot=self._build_root) if artifacts_file is not None: artifacts_json = json.loads(artifacts_file) extra_upload_urls = artifacts_json.get('extra_upload_urls', []) upload_urls += [ self.UniqifyArchiveUrl(url) for url in extra_upload_urls ] return upload_urls
def LoadArtifactsList(self, board, image_dir): """Load the list of artifacts to upload for this board. It attempts to load a JSON file, scripts/artifacts.json, from the overlay directories for this board. This file specifies the artifacts to generate, if it can't be found, it will use a default set that uploads every .bin file as a .tar.xz file. See BuildStandaloneArchive in cbuildbot_commands.py for format docs. """ custom_artifacts_file = portage_util.ReadOverlayFile( 'scripts/artifacts.json', board=board) artifacts = None if custom_artifacts_file is not None: json_file = json.loads(custom_artifacts_file) artifacts = json_file.get('artifacts') if artifacts is None: artifacts = [] for image_file in glob.glob(os.path.join(image_dir, '*.bin')): basename = os.path.basename(image_file) info = { 'input': [basename], 'archive': 'tar', 'compress': 'xz' } artifacts.append(info) # We add the dlc folder (if exists) as artifact so we can copy all DLC # artifacts as is. if os.path.isdir(os.path.join(image_dir, 'dlc')): artifacts.append({'input': ['dlc']}) for artifact in artifacts: # Resolve the (possible) globs in the input list, and store # the actual set of files to use in 'paths' paths = [] for s in artifact['input']: glob_paths = glob.glob(os.path.join(image_dir, s)) if not glob_paths: logging.warning('No artifacts generated for input: %s', s) else: for path in glob_paths: paths.append(os.path.relpath(path, image_dir)) artifact['paths'] = paths self.artifacts = artifacts
def _LoadTests(self): """Loads the tests to run from <overlay>/scripts/gce_tests.json. If the JSON file exists, loads the tests and flags to create instance for each test with. The JSON file should contain a "tests" object, which is an array of objects, each of which has only two keys: "name" and "flags". "name" could be any valid Autotest test name, or a suite name, in the form of "suite:<suite_name>", e.g., "suite:gce-smoke". "flags" is a JSON object whose members must be valid proterties of the GCE Instance Resource, as specificed at: https://cloud.google.com/compute/docs/reference/latest/instances#resource. These flags will be used to create instances. Each flag must strictly follow the property schema as defined in the Instance Resource. Failure to do so will result in instance creation failures. Note that a dedicated instance will be created for every test object specified in scripts/gce_tests.json. So group test cases that require similar instance properties together as suites whenever possible. An example scripts/gce_tests.json may look like: { "tests": [ { "name": "suite:gce-smoke", "flags": [] }, { "name": "suite:cloud-init", "flags": { "description": "Test instance", "metadata": { "items": [ { "key": "fake_key", "value": "fake_value" } ] } } } ] } If the JSON file does not exist, the 'gce-smoke' suite will be used to verify the image. """ # Defaults to run the gce-smoke suite if no custom tests are given. tests = [dict(name="suite:gce-smoke", flags=dict())] custom_tests = None try: custom_tests = portage_util.ReadOverlayFile( 'scripts/gce_tests.json', board=self.board) except portage_util.MissingOverlayException as e: logging.warn('Board overlay not found. Error: %r', e) if custom_tests is not None: if self.board not in constants.TRUSTED_BOARDS: logging.warn('Custom tests and flags are not allowed for this board ' '(%s)!', self.board) else: # Read the list of tests. try: json_file = json.loads(custom_tests) tests = json_file.get('tests') except ValueError as e: logging.warn('scripts/gce_tests.json contains invalid JSON content. ' 'Default tests will be run and default flags will be ' 'used to create instances. Error: %r', e) self.tests = tests