def _ListBuilds(self, branch, target, page_token):
"""List builds as build items."""
client = self._GetClient()
req = client.build().list(
buildType='submitted',
buildAttemptStatus='complete',
branch=branch,
target=target,
pageToken=page_token)
res = req.execute(num_retries=constant.NUM_RETRIES)
build_items = [
base.BuildItem(
name=b['buildId'],
path='%s/%s/%s' % (branch, target, b['buildId']),
is_file=False,
size=None,
timestamp=_ParseTimestamp(b.get('creationTimestamp')))
for b in res.get('builds', [])
]
# Add a latest folder to the first page
if not page_token and build_items:
latest_buid_item = base.BuildItem(
name=LATEST,
path='%s/%s/%s' % (branch, target, LATEST),
is_file=False,
size=None,
timestamp=None,
description=LATEST_BUILD_ITEM_DESCRIPTION)
build_items.insert(0, latest_buid_item)
next_page_token = res.get('nextPageToken')
return (build_items, next_page_token)
def testBuildUrl(self):
"""Tests that build channel URLs can be generated."""
item = plugins.BuildItem(name='z z z.txt',
path='a/b/c/z z z.txt',
is_file=True)
encoded_url = build.BuildUrl('local_file_store', item)
self.assertEqual(encoded_url,
'mtt:///local_file_store/a/b/c/z%20z%20z.txt')
item = plugins.BuildItem(name='d/e.txt',
path='a/b/c/d/e.txt',
is_file=True)
encoded_url = build.BuildUrl('local_file_store', item)
self.assertEqual(encoded_url,
'mtt:///local_file_store/a/b/c/d%2Fe.txt')
def testDownloadFile(self, mock_next_chunk, mock_get_build_item):
"""Test downloading a file with multiple chunks."""
provider = gcs.GCSBuildProvider()
provider._client = mock.MagicMock()
path = 'bucket/fake/path/kitten.png'
mock_get_build_item.return_value = base.BuildItem(
name='kitten.png',
path='bucket/fake/path/kitten.png',
is_file=True,
size=0,
timestamp=None)
# Mock downloader that processes two chunks of data
mock_progress = iter([
(b'hello', mock.MagicMock(resumable_progress=5,
total_size=10), False),
(b'world', mock.MagicMock(resumable_progress=10,
total_size=10), True),
])
def _next_chunk(self, **_):
value, status, done = next(mock_progress)
self._fd.write(value)
return status, done
mock_next_chunk.side_effect = _next_chunk
result = list(provider.DownloadFile(path))
provider._client.objects().get_media.assert_called_with(
bucket='bucket', object='fake/path/kitten.png')
expected = [
file_util.FileChunk(data=b'hello', offset=5, total_size=10),
file_util.FileChunk(data=b'world', offset=10, total_size=10)
]
self.assertEqual(expected, result)
def _ListBuildArtifacts(self, branch, target, build_id, page_token):
"""Lists build artifacts as build items."""
client = self._GetClient()
is_latest_build = build_id == LATEST
# If selected folder is latest folder, make api call to get the build_id
# for latest folder
if is_latest_build:
build_id = self._GetLatestBuild(branch=branch, target=target)['buildId']
req = client.buildartifact().list(
target=target,
buildId=build_id,
attemptId='latest',
pageToken=page_token)
res = req.execute(num_retries=constant.NUM_RETRIES)
artifacts = res.get('artifacts', [])
# if in latest folder, change its path to contain latest
if is_latest_build:
build_id = LATEST
build_items = [
base.BuildItem(
name=a['name'],
path='%s/%s/%s/%s' % (branch, target, build_id, a['name']),
is_file=True,
size=int(a['size']),
timestamp=_ParseTimestamp(a.get('lastModifiedTime')))
for a in artifacts if 'name' in a
]
next_page_token = res.get('nextPageToken')
return (build_items, next_page_token)
def _GetBranch(self, name):
"""Returns a branch as a build item."""
client = self._GetClient()
req = client.branch().get(resourceId=name, fields=['name'])
res = req.execute(num_retries=constant.NUM_RETRIES)
return base.BuildItem(
name=res['name'], path=res['name'], is_file=False, size=None,
timestamp=None)
def _GetTarget(self, branch, name):
"""Returns a build target as a build item."""
client = self._GetClient()
req = client.target().get(branch=branch, resourceId=name)
res = req.execute(num_retries=constant.NUM_RETRIES)
return base.BuildItem(
name=res['name'], path='%s/%s' % (branch, res['name']), is_file=False,
size=None, timestamp=None)
def testListBuildItems(self, mock_channel):
# create placeholders build items to return from channel
build_items = [
plugins.BuildItem(name='item1', path=u'path1', is_file=True),
plugins.BuildItem(name='item2', path=u'path2', is_file=True)
]
mock_channel.return_value = build_items, 'next_page_token'
config = self._CreateMockBuildChannel()
res = self.app.get('/_ah/api/mtt/v1/build_channels/%s/build_items' %
config.key.id())
build_item_list = protojson.decode_message(messages.BuildItemList,
res.body)
# verify same build items are returned
self.assertEqual('next_page_token', build_item_list.next_page_token)
self.assertItemsEqual(
['item1', 'item2'],
[item.name for item in build_item_list.build_items])
def FindFile(build_channel_id, path, filename=None):
"""Finds a file build item under a given path recursively.
Args:
build_channel_id: a build channel ID.
path: a build item path.
filename: a filename. Can containd wildcard characters (*?).
Returns:
a plugins.BuildItem object.
"""
build_channel = GetBuildChannel(build_channel_id)
if not build_channel:
raise ValueError('Build channel [%s] does not exist' % build_channel_id)
if path:
build_item = build_channel.GetBuildItem(path)
else:
# Path can be none when file are at first level into the build channel
# e.g. mtt:///google_drive/file.txt, in this case google_drive in the
# id, and file.txt is filename, but path is none.
build_item = plugins.BuildItem(name=None, path=None, is_file=False)
if not filename:
return build_item
# If a given path is for a director and a filename is given, recursively
# search the first file that matches.
while build_item and not build_item.is_file:
next_build_item = None
# If filename has wildcard characters
if any([c in filename for c in WILDCARD_CHARS]):
for child_item in _BuildItemIterator(
build_channel, build_item.path, item_type=BuildItemType.FILE):
if child_item.is_file and fnmatch.fnmatch(child_item.name, filename):
next_build_item = child_item
break
else:
child_item = build_channel.GetBuildItem(
os.path.join(build_item.path or '', filename))
if child_item and child_item.is_file:
next_build_item = child_item
if not next_build_item:
# We assume the first build item returned is the latest one.
for child_item in _BuildItemIterator(
build_channel,
build_item.path,
item_type=BuildItemType.DIRECTORY):
next_build_item = child_item
break
build_item = next_build_item
return build_item
def testDownloadFile_withInvalidFile(self, mock_get_build_item):
"""Try to download folder instead of file."""
provider = google_drive.GoogleDriveBuildProvider()
path = 'fake/path/'
fake_build_item = base.BuildItem(name='fake/path/',
path='fake/path/',
is_file=False,
size=0,
timestamp=None)
mock_get_build_item.return_value = fake_build_item
with self.assertRaises(errors.FileNotFoundError) as e:
list(provider.DownloadFile(path))
self.assertEqual(e.exception.message,
google_drive._FILE_NOT_FOUND_ERROR % path)
def _ListBranches(self, page_token):
"""List branches as build items."""
client = self._GetClient()
req = client.branch().list(
fields='branches/name,nextPageToken', pageToken=page_token)
res = req.execute(num_retries=constant.NUM_RETRIES)
build_items = [
base.BuildItem(
name=b['name'], path=b['name'], is_file=False, size=None,
timestamp=None)
for b in res.get('branches', [])
]
next_page_token = res.get('nextPageToken')
return (build_items, next_page_token)
def testDownloadFile_withInvalidFile(self, mock_get_build_item):
"""Try to download folder instead of file."""
provider = gcs.GCSBuildProvider()
path = 'bucket/fake/path/'
fake_build_item = base.BuildItem(name='path/',
path='bucket/fake/path/',
is_file=False,
size=0,
timestamp=None)
mock_get_build_item.return_value = fake_build_item
with self.assertRaises(errors.FileNotFoundError) as e:
list(provider.DownloadFile(path))
self.assertEqual(e.exception.message,
'Build item bucket/fake/path/ does not exist')
def _GetBuild(self, branch, target, build_id):
"""Returns a build as a build item."""
if build_id == LATEST:
res = self._GetLatestBuild(branch=branch, target=target)
else:
client = self._GetClient()
req = client.build().get(target=target, buildId=build_id)
res = req.execute(num_retries=constant.NUM_RETRIES)
return base.BuildItem(
name=res['buildId'],
path='%s/%s/%s' % (branch, target, res['buildId']),
is_file=False,
size=None,
timestamp=_ParseTimestamp(res.get('creationTimestamp')))
def _ListTargets(self, branch, page_token):
"""List build targets as build items."""
client = self._GetClient()
req = client.target().list(
branch=branch, fields='targets/name,nextPageToken',
pageToken=page_token)
res = req.execute(num_retries=constant.NUM_RETRIES)
build_items = [
base.BuildItem(
name=t['name'], path='%s/%s' % (branch, t['name']), is_file=False,
size=None, timestamp=None)
for t in res.get('targets', [])
]
next_page_token = res.get('nextPageToken')
return (build_items, next_page_token)
def _GetBuildArtifact(self, branch, target, build_id, name):
"""Returns a build artifact as a build item."""
# if build_id is 'latest', make sure find the latest resource
if build_id == LATEST:
build_id = self._GetLatestBuild(branch=branch, target=target)['buildId']
client = self._GetClient()
req = client.buildartifact().get(
target=target, buildId=build_id, attemptId='latest', resourceId=name)
res = req.execute(num_retries=constant.NUM_RETRIES)
return base.BuildItem(
name=res['name'],
path='%s/%s/%s/%s' % (branch, target, build_id, res['name']),
is_file=True,
size=int(res['size']),
timestamp=_ParseTimestamp(res.get('lastModifiedTime')))
def testLookupBuildItem(self):
config = self._CreateMockBuildChannel(name='android',
provider='Android')
url = 'mtt:///%s/path/to/file.ext' % config.key.id()
build_item = plugins.BuildItem(name='foo',
path='zzz/bar/foo',
is_file=True,
size=1234,
timestamp=datetime.datetime.utcnow())
AndroidBuildProvider.mock.GetBuildItem.return_value = build_item
res = self.app.get(
'/_ah/api/mtt/v1/build_channels/build_item_lookup?url=%s' %
(urllib.parse.quote(url)))
build_item_msg = protojson.decode_message(messages.BuildItem, res.body)
AndroidBuildProvider.mock.GetBuildItem.assert_called_once_with(
'path/to/file.ext')
self.assertEqual(messages.Convert(build_item, messages.BuildItem),
build_item_msg)
def setUp(self):
super(TestPlanApiTest, self).setUp(test_plan_api.TestPlanApi)
self.mock_test = ndb_models.Test(name='mock_test', command='command')
self.mock_test.put()
self.mock_test_id = str(self.mock_test.key.id())
self.mock_build_channel = ndb_models.BuildChannelConfig(
id=str(uuid.uuid4()),
name='mock_build_channel',
provider_name='mock_build_provider')
self.mock_build_channel.put()
self.mock_build_channel_id = self.mock_build_channel.key.id()
self.mock_device_action = ndb_models.DeviceAction(
name='mock_device_action')
self.mock_device_action.put()
self.mock_device_action_id = str(self.mock_device_action.key.id())
build_item = base.BuildItem(name='zz',
path='/foo/bar/zz',
is_file=True)
self.mock_test_resource_url = build.BuildUrl(
self.mock_build_channel_id, build_item)
def _ConvertFileToBuildItem(self, file_item, path):
"""Convert a file_item to a build_item.
Args:
file_item: a google drive file object
path: path to the file (e.g folderB/folderA/, folderB/cat.img)
Returns:
base.BuildItem
"""
is_file = file_item['mimeType'] != _FOLDER_TYPE
name = file_item['name']
if is_file:
timestamp = _ParseTimeStamp(file_item['modifiedTime'])
else:
name = name + _PATH_DELIMITER
timestamp = None
return base.BuildItem(
name=name,
path=path,
is_file=is_file,
size=int(file_item.get('size', 0)),
timestamp=timestamp)
