def _GetUploadUrls(self, filename, builder_run=None):
"""Returns a list of all urls for which to upload filename to.
Args:
filename: The filename of the file we want to upload.
builder_run: builder_run object from which to get the board, base upload
url, and bot_id. If none, this stage's values.
"""
board = None
urls = [self.upload_url]
bot_id = self._bot_id
if builder_run:
urls = [builder_run.GetArchive().upload_url]
bot_id = builder_run.GetArchive().bot_id
if (builder_run.config['boards']
and len(builder_run.config['boards']) == 1):
board = builder_run.config['boards'][0]
if (not self._IsInUploadBlacklist(filename)
and (hasattr(self, '_current_board') or board)):
if self._run.config.pre_cq:
# Do not load artifacts.json for pre-cq configs. This is a
# workaround for crbug.com/440167.
return urls
board = board or self._current_board
custom_artifacts_file = portage_util.ReadOverlayFile(
'scripts/artifacts.json', board=board)
if custom_artifacts_file is not None:
json_file = json.loads(custom_artifacts_file)
for url in json_file.get('extra_upload_urls', []):
urls.append('/'.join([url, bot_id, self.version]))
return urls
def _GetUploadUrls(self, filename, builder_run=None):
"""Returns a list of all urls for which to upload filename to.
Args:
filename: The filename of the file we want to upload.
builder_run: builder_run object from which to get the board, base upload
url, and bot_id. If none, this stage's values.
"""
board = None
urls = [self.upload_url]
bot_id = self._bot_id
if builder_run:
urls = [builder_run.GetArchive().upload_url]
bot_id = builder_run.GetArchive().bot_id
if (builder_run.config['boards']
and len(builder_run.config['boards']) == 1):
board = builder_run.config['boards'][0]
if (not self._IsInUploadBlacklist(filename)
and (hasattr(self, '_current_board') or board)):
board = board or self._current_board
custom_artifacts_file = portage_util.ReadOverlayFile(
'scripts/artifacts.json', board=board)
if custom_artifacts_file is not None:
json_file = json.loads(custom_artifacts_file)
for url in json_file.get('extra_upload_urls', []):
# Moblab users do not need the paladin, pfq or trybot
# builds, filter those bots from extra uploads.
if self._FilterBuildFromMoblab(url, bot_id):
continue
urls.append('/'.join([url, bot_id, self.version]))
return urls
def testReadOverlayFileOrder(self):
"""Verify that the boards are examined in the right order."""
m = self.PatchObject(os.path, 'isfile', return_value=False)
portage_util.ReadOverlayFile('test', self.PUBLIC, self.PUB_PRIV,
self.tempdir)
read_overlays = [x[0][0][:-5] for x in m.call_args_list]
overlays = [x for x in reversed(self.overlays[self.PUB_PRIV][self.PUBLIC])]
self.assertEqual(read_overlays, overlays)
def GetDummyArchiveUrls(self):
"""Returns upload URLs for dummy artifacts based on artifacts.json."""
upload_urls = [self.dummy_archive_url]
artifacts_file = portage_util.ReadOverlayFile(
'scripts/artifacts.json',
board=self._current_board,
buildroot=self._build_root)
if artifacts_file is not None:
artifacts_json = json.loads(artifacts_file)
extra_upload_urls = artifacts_json.get('extra_upload_urls', [])
upload_urls += [
self.UniqifyArchiveUrl(url) for url in extra_upload_urls
]
return upload_urls
def LoadArtifactsList(self, board, image_dir):
"""Load the list of artifacts to upload for this board.
It attempts to load a JSON file, scripts/artifacts.json, from the
overlay directories for this board. This file specifies the artifacts
to generate, if it can't be found, it will use a default set that
uploads every .bin file as a .tar.xz file.
See BuildStandaloneArchive in cbuildbot_commands.py for format docs.
"""
custom_artifacts_file = portage_util.ReadOverlayFile(
'scripts/artifacts.json', board=board)
artifacts = None
if custom_artifacts_file is not None:
json_file = json.loads(custom_artifacts_file)
artifacts = json_file.get('artifacts')
if artifacts is None:
artifacts = []
for image_file in glob.glob(os.path.join(image_dir, '*.bin')):
basename = os.path.basename(image_file)
info = {
'input': [basename],
'archive': 'tar',
'compress': 'xz'
}
artifacts.append(info)
# We add the dlc folder (if exists) as artifact so we can copy all DLC
# artifacts as is.
if os.path.isdir(os.path.join(image_dir, 'dlc')):
artifacts.append({'input': ['dlc']})
for artifact in artifacts:
# Resolve the (possible) globs in the input list, and store
# the actual set of files to use in 'paths'
paths = []
for s in artifact['input']:
glob_paths = glob.glob(os.path.join(image_dir, s))
if not glob_paths:
logging.warning('No artifacts generated for input: %s', s)
else:
for path in glob_paths:
paths.append(os.path.relpath(path, image_dir))
artifact['paths'] = paths
self.artifacts = artifacts
def _LoadTests(self):
"""Loads the tests to run from <overlay>/scripts/gce_tests.json.
If the JSON file exists, loads the tests and flags to create instance for
each test with. The JSON file should contain a "tests" object, which is an
array of objects, each of which has only two keys: "name" and "flags".
"name" could be any valid Autotest test name, or a suite name, in the form
of "suite:<suite_name>", e.g., "suite:gce-smoke".
"flags" is a JSON object whose members must be valid proterties of the GCE
Instance Resource, as specificed at:
https://cloud.google.com/compute/docs/reference/latest/instances#resource.
These flags will be used to create instances. Each flag must strictly follow
the property schema as defined in the Instance Resource. Failure to do so
will result in instance creation failures.
Note that a dedicated instance will be created for every test object
specified in scripts/gce_tests.json. So group test cases that require
similar instance properties together as suites whenever possible.
An example scripts/gce_tests.json may look like:
{
"tests": [
{
"name": "suite:gce-smoke",
"flags": []
},
{
"name": "suite:cloud-init",
"flags": {
"description": "Test instance",
"metadata": {
"items": [
{
"key": "fake_key",
"value": "fake_value"
}
]
}
}
}
]
}
If the JSON file does not exist, the 'gce-smoke' suite will be used to
verify the image.
"""
# Defaults to run the gce-smoke suite if no custom tests are given.
tests = [dict(name="suite:gce-smoke", flags=dict())]
custom_tests = None
try:
custom_tests = portage_util.ReadOverlayFile(
'scripts/gce_tests.json', board=self.board)
except portage_util.MissingOverlayException as e:
logging.warn('Board overlay not found. Error: %r', e)
if custom_tests is not None:
if self.board not in constants.TRUSTED_BOARDS:
logging.warn('Custom tests and flags are not allowed for this board '
'(%s)!', self.board)
else:
# Read the list of tests.
try:
json_file = json.loads(custom_tests)
tests = json_file.get('tests')
except ValueError as e:
logging.warn('scripts/gce_tests.json contains invalid JSON content. '
'Default tests will be run and default flags will be '
'used to create instances. Error: %r', e)
self.tests = tests
