def _ListBuilds(self, branch, target, page_token): """List builds as build items.""" client = self._GetClient() req = client.build().list( buildType='submitted', buildAttemptStatus='complete', branch=branch, target=target, pageToken=page_token) res = req.execute(num_retries=constant.NUM_RETRIES) build_items = [ base.BuildItem( name=b['buildId'], path='%s/%s/%s' % (branch, target, b['buildId']), is_file=False, size=None, timestamp=_ParseTimestamp(b.get('creationTimestamp'))) for b in res.get('builds', []) ] # Add a latest folder to the first page if not page_token and build_items: latest_buid_item = base.BuildItem( name=LATEST, path='%s/%s/%s' % (branch, target, LATEST), is_file=False, size=None, timestamp=None, description=LATEST_BUILD_ITEM_DESCRIPTION) build_items.insert(0, latest_buid_item) next_page_token = res.get('nextPageToken') return (build_items, next_page_token)
def testBuildUrl(self): """Tests that build channel URLs can be generated.""" item = plugins.BuildItem(name='z z z.txt', path='a/b/c/z z z.txt', is_file=True) encoded_url = build.BuildUrl('local_file_store', item) self.assertEqual(encoded_url, 'mtt:///local_file_store/a/b/c/z%20z%20z.txt') item = plugins.BuildItem(name='d/e.txt', path='a/b/c/d/e.txt', is_file=True) encoded_url = build.BuildUrl('local_file_store', item) self.assertEqual(encoded_url, 'mtt:///local_file_store/a/b/c/d%2Fe.txt')
def testDownloadFile(self, mock_next_chunk, mock_get_build_item): """Test downloading a file with multiple chunks.""" provider = gcs.GCSBuildProvider() provider._client = mock.MagicMock() path = 'bucket/fake/path/kitten.png' mock_get_build_item.return_value = base.BuildItem( name='kitten.png', path='bucket/fake/path/kitten.png', is_file=True, size=0, timestamp=None) # Mock downloader that processes two chunks of data mock_progress = iter([ (b'hello', mock.MagicMock(resumable_progress=5, total_size=10), False), (b'world', mock.MagicMock(resumable_progress=10, total_size=10), True), ]) def _next_chunk(self, **_): value, status, done = next(mock_progress) self._fd.write(value) return status, done mock_next_chunk.side_effect = _next_chunk result = list(provider.DownloadFile(path)) provider._client.objects().get_media.assert_called_with( bucket='bucket', object='fake/path/kitten.png') expected = [ file_util.FileChunk(data=b'hello', offset=5, total_size=10), file_util.FileChunk(data=b'world', offset=10, total_size=10) ] self.assertEqual(expected, result)
def _ListBuildArtifacts(self, branch, target, build_id, page_token): """Lists build artifacts as build items.""" client = self._GetClient() is_latest_build = build_id == LATEST # If selected folder is latest folder, make api call to get the build_id # for latest folder if is_latest_build: build_id = self._GetLatestBuild(branch=branch, target=target)['buildId'] req = client.buildartifact().list( target=target, buildId=build_id, attemptId='latest', pageToken=page_token) res = req.execute(num_retries=constant.NUM_RETRIES) artifacts = res.get('artifacts', []) # if in latest folder, change its path to contain latest if is_latest_build: build_id = LATEST build_items = [ base.BuildItem( name=a['name'], path='%s/%s/%s/%s' % (branch, target, build_id, a['name']), is_file=True, size=int(a['size']), timestamp=_ParseTimestamp(a.get('lastModifiedTime'))) for a in artifacts if 'name' in a ] next_page_token = res.get('nextPageToken') return (build_items, next_page_token)
def _GetBranch(self, name): """Returns a branch as a build item.""" client = self._GetClient() req = client.branch().get(resourceId=name, fields=['name']) res = req.execute(num_retries=constant.NUM_RETRIES) return base.BuildItem( name=res['name'], path=res['name'], is_file=False, size=None, timestamp=None)
def _GetTarget(self, branch, name): """Returns a build target as a build item.""" client = self._GetClient() req = client.target().get(branch=branch, resourceId=name) res = req.execute(num_retries=constant.NUM_RETRIES) return base.BuildItem( name=res['name'], path='%s/%s' % (branch, res['name']), is_file=False, size=None, timestamp=None)
def testListBuildItems(self, mock_channel): # create placeholders build items to return from channel build_items = [ plugins.BuildItem(name='item1', path=u'path1', is_file=True), plugins.BuildItem(name='item2', path=u'path2', is_file=True) ] mock_channel.return_value = build_items, 'next_page_token' config = self._CreateMockBuildChannel() res = self.app.get('/_ah/api/mtt/v1/build_channels/%s/build_items' % config.key.id()) build_item_list = protojson.decode_message(messages.BuildItemList, res.body) # verify same build items are returned self.assertEqual('next_page_token', build_item_list.next_page_token) self.assertItemsEqual( ['item1', 'item2'], [item.name for item in build_item_list.build_items])
def FindFile(build_channel_id, path, filename=None): """Finds a file build item under a given path recursively. Args: build_channel_id: a build channel ID. path: a build item path. filename: a filename. Can containd wildcard characters (*?). Returns: a plugins.BuildItem object. """ build_channel = GetBuildChannel(build_channel_id) if not build_channel: raise ValueError('Build channel [%s] does not exist' % build_channel_id) if path: build_item = build_channel.GetBuildItem(path) else: # Path can be none when file are at first level into the build channel # e.g. mtt:///google_drive/file.txt, in this case google_drive in the # id, and file.txt is filename, but path is none. build_item = plugins.BuildItem(name=None, path=None, is_file=False) if not filename: return build_item # If a given path is for a director and a filename is given, recursively # search the first file that matches. while build_item and not build_item.is_file: next_build_item = None # If filename has wildcard characters if any([c in filename for c in WILDCARD_CHARS]): for child_item in _BuildItemIterator( build_channel, build_item.path, item_type=BuildItemType.FILE): if child_item.is_file and fnmatch.fnmatch(child_item.name, filename): next_build_item = child_item break else: child_item = build_channel.GetBuildItem( os.path.join(build_item.path or '', filename)) if child_item and child_item.is_file: next_build_item = child_item if not next_build_item: # We assume the first build item returned is the latest one. for child_item in _BuildItemIterator( build_channel, build_item.path, item_type=BuildItemType.DIRECTORY): next_build_item = child_item break build_item = next_build_item return build_item
def testDownloadFile_withInvalidFile(self, mock_get_build_item): """Try to download folder instead of file.""" provider = google_drive.GoogleDriveBuildProvider() path = 'fake/path/' fake_build_item = base.BuildItem(name='fake/path/', path='fake/path/', is_file=False, size=0, timestamp=None) mock_get_build_item.return_value = fake_build_item with self.assertRaises(errors.FileNotFoundError) as e: list(provider.DownloadFile(path)) self.assertEqual(e.exception.message, google_drive._FILE_NOT_FOUND_ERROR % path)
def _ListBranches(self, page_token): """List branches as build items.""" client = self._GetClient() req = client.branch().list( fields='branches/name,nextPageToken', pageToken=page_token) res = req.execute(num_retries=constant.NUM_RETRIES) build_items = [ base.BuildItem( name=b['name'], path=b['name'], is_file=False, size=None, timestamp=None) for b in res.get('branches', []) ] next_page_token = res.get('nextPageToken') return (build_items, next_page_token)
def testDownloadFile_withInvalidFile(self, mock_get_build_item): """Try to download folder instead of file.""" provider = gcs.GCSBuildProvider() path = 'bucket/fake/path/' fake_build_item = base.BuildItem(name='path/', path='bucket/fake/path/', is_file=False, size=0, timestamp=None) mock_get_build_item.return_value = fake_build_item with self.assertRaises(errors.FileNotFoundError) as e: list(provider.DownloadFile(path)) self.assertEqual(e.exception.message, 'Build item bucket/fake/path/ does not exist')
def _GetBuild(self, branch, target, build_id): """Returns a build as a build item.""" if build_id == LATEST: res = self._GetLatestBuild(branch=branch, target=target) else: client = self._GetClient() req = client.build().get(target=target, buildId=build_id) res = req.execute(num_retries=constant.NUM_RETRIES) return base.BuildItem( name=res['buildId'], path='%s/%s/%s' % (branch, target, res['buildId']), is_file=False, size=None, timestamp=_ParseTimestamp(res.get('creationTimestamp')))
def _ListTargets(self, branch, page_token): """List build targets as build items.""" client = self._GetClient() req = client.target().list( branch=branch, fields='targets/name,nextPageToken', pageToken=page_token) res = req.execute(num_retries=constant.NUM_RETRIES) build_items = [ base.BuildItem( name=t['name'], path='%s/%s' % (branch, t['name']), is_file=False, size=None, timestamp=None) for t in res.get('targets', []) ] next_page_token = res.get('nextPageToken') return (build_items, next_page_token)
def _GetBuildArtifact(self, branch, target, build_id, name): """Returns a build artifact as a build item.""" # if build_id is 'latest', make sure find the latest resource if build_id == LATEST: build_id = self._GetLatestBuild(branch=branch, target=target)['buildId'] client = self._GetClient() req = client.buildartifact().get( target=target, buildId=build_id, attemptId='latest', resourceId=name) res = req.execute(num_retries=constant.NUM_RETRIES) return base.BuildItem( name=res['name'], path='%s/%s/%s/%s' % (branch, target, build_id, res['name']), is_file=True, size=int(res['size']), timestamp=_ParseTimestamp(res.get('lastModifiedTime')))
def testLookupBuildItem(self): config = self._CreateMockBuildChannel(name='android', provider='Android') url = 'mtt:///%s/path/to/file.ext' % config.key.id() build_item = plugins.BuildItem(name='foo', path='zzz/bar/foo', is_file=True, size=1234, timestamp=datetime.datetime.utcnow()) AndroidBuildProvider.mock.GetBuildItem.return_value = build_item res = self.app.get( '/_ah/api/mtt/v1/build_channels/build_item_lookup?url=%s' % (urllib.parse.quote(url))) build_item_msg = protojson.decode_message(messages.BuildItem, res.body) AndroidBuildProvider.mock.GetBuildItem.assert_called_once_with( 'path/to/file.ext') self.assertEqual(messages.Convert(build_item, messages.BuildItem), build_item_msg)
def setUp(self): super(TestPlanApiTest, self).setUp(test_plan_api.TestPlanApi) self.mock_test = ndb_models.Test(name='mock_test', command='command') self.mock_test.put() self.mock_test_id = str(self.mock_test.key.id()) self.mock_build_channel = ndb_models.BuildChannelConfig( id=str(uuid.uuid4()), name='mock_build_channel', provider_name='mock_build_provider') self.mock_build_channel.put() self.mock_build_channel_id = self.mock_build_channel.key.id() self.mock_device_action = ndb_models.DeviceAction( name='mock_device_action') self.mock_device_action.put() self.mock_device_action_id = str(self.mock_device_action.key.id()) build_item = base.BuildItem(name='zz', path='/foo/bar/zz', is_file=True) self.mock_test_resource_url = build.BuildUrl( self.mock_build_channel_id, build_item)
def _ConvertFileToBuildItem(self, file_item, path): """Convert a file_item to a build_item. Args: file_item: a google drive file object path: path to the file (e.g folderB/folderA/, folderB/cat.img) Returns: base.BuildItem """ is_file = file_item['mimeType'] != _FOLDER_TYPE name = file_item['name'] if is_file: timestamp = _ParseTimeStamp(file_item['modifiedTime']) else: name = name + _PATH_DELIMITER timestamp = None return base.BuildItem( name=name, path=path, is_file=is_file, size=int(file_item.get('size', 0)), timestamp=timestamp)