def testListFiles(self, mock_urlopen):
"""Tests that nested files can be listed."""
data = json.dumps([
{
'name': 'foo',
'path': 'path/foo',
'size': 4096,
'type': 'DIRECTORY',
'update_time': 631152000000
},
{
'name': 'bar',
'path': 'path/bar',
'size': 123456,
'type': 'FILE',
'update_time': 946684800000
},
])
mock_urlopen.return_value = io.BytesIO(six.ensure_binary(data))
files = file_util.RemoteFileHandle('file:///root/path').ListFiles()
mock_urlopen.assert_called_with('http://file_server:8006/dir/path')
self.assertEqual(files, [
file_util.FileInfo(url='file:///root/path/foo',
is_file=False,
total_size=4096,
timestamp=datetime.datetime(1990, 1, 1)),
file_util.FileInfo(url='file:///root/path/bar',
is_file=True,
total_size=123456,
timestamp=datetime.datetime(2000, 1, 1))
])
def testCleanTestResourceCache_directory(self, mock_handle_factory):
"""Tests that directories are ignored when cleaning cache."""
mock_handle_factory.return_value.ListFiles.return_value = [
file_util.FileInfo(url='url',
is_file=False,
timestamp=datetime.datetime.utcnow() -
datetime.timedelta(hours=1))
]
# Directory is not deleted
download_util.CleanTestResourceCache()
mock_handle_factory.return_value.Delete.assert_not_called()
def testCleanTestResourceCache(self, mock_handle_factory):
"""Tests that cached test resources can be deleted."""
mock_handle_factory.return_value.ListFiles.return_value = [
file_util.FileInfo(url='url',
is_file=True,
timestamp=datetime.datetime.utcnow() -
datetime.timedelta(hours=1))
]
# File is deleted
download_util.CleanTestResourceCache()
mock_handle_factory.return_value.Delete.assert_called()
def testCleanTestResourceCache_recentlyUpdated(self, mock_handle_factory):
"""Tests that recently updated files are ignored when cleaning cache."""
mock_handle_factory.return_value.ListFiles.return_value = [
file_util.FileInfo(url='url',
is_file=True,
timestamp=datetime.datetime.utcnow())
]
# Recently updated file is not deleted (timestamp > min_access_time)
min_access_time = datetime.datetime.utcnow() - datetime.timedelta(
hours=1)
download_util.CleanTestResourceCache(min_access_time=min_access_time)
mock_handle_factory.return_value.Delete.assert_not_called()
def testDownloadResource_outOfDate(self, mock_handle_factory,
mock_download_file):
# Source URL nad mock handle (with size and timestamp)
url = 'http://www.foo.com/bar/zzz.ext'
src_handle = mock.MagicMock()
src_handle.Info.return_value = file_util.FileInfo(
123, 'type', datetime.datetime(2000, 1, 1))
# Destination URL and mock handle (timestamp out of date)
dst_url = download_util.GetCacheUrl(url)
dst_handle = mock.MagicMock()
dst_handle.Info.return_value = file_util.FileInfo(
123, 'type', datetime.datetime(1990, 1, 1))
# Return right handle based on URL
mock_handle_factory.side_effect = {
url: src_handle,
dst_url: dst_handle
}.get
cache_url = download_util.DownloadResource(url)
self.assertEqual(dst_url, cache_url)
# downloaded file despite cached file
mock_download_file.assert_called_with(url)
def testDownloadResource_existingFile(self, mock_handle_factory,
mock_download_file):
# Source URL nad mock handle (with size and timestamp)
url = 'http://www.foo.com/bar/zzz.ext'
src_handle = mock.MagicMock()
src_handle.Info.return_value = file_util.FileInfo(
url=url, total_size=123, timestamp=datetime.datetime(1990, 1, 1))
# Destination URL and mock handle (valid size and up-to-date timestamp)
dst_url = download_util.GetCacheUrl(url)
dst_handle = mock.MagicMock()
dst_handle.Info.return_value = file_util.FileInfo(
url=dst_url,
total_size=123,
timestamp=datetime.datetime(1990, 1, 1))
# Return right handle based on URL
mock_handle_factory.side_effect = {
url: src_handle,
dst_url: dst_handle
}.get
cache_url = download_util.DownloadResource(url)
self.assertEqual(dst_url, cache_url)
# Download skipped due to cached file
mock_download_file.assert_not_called()
def testCleanTestResourceCache_recentlyUsed(self, mock_handle_factory):
"""Tests that recently used files are ignored when cleaning cache."""
test_run = ndb_models.TestRun(test_resources=[
ndb_models.TestResourceObj(name='resource', url='resource_url')
])
test_run.put()
# Cached resource is old but recently referenced
mock_handle_factory.return_value.ListFiles.return_value = [
file_util.FileInfo(url=download_util.GetCacheUrl('resource_url'),
is_file=True,
timestamp=datetime.datetime.utcnow() -
datetime.timedelta(days=1))
]
# Recently used file is not deleted (test_run > min_access_time)
min_access_time = datetime.datetime.utcnow() - datetime.timedelta(
hours=1)
download_util.CleanTestResourceCache(min_access_time=min_access_time)
mock_handle_factory.return_value.Delete.assert_not_called()
def __init__(self, url):
"""Constructs a download context for a given resource URL."""
self.url = url
self.cache_url = GetCacheUrl(url)
build_channel, build_item_path = build.FindBuildChannel(url)
if not build_channel:
# URL does not match any build channel, use default download context
self.source_url = url
self.download_fn = file_util.DownloadFile
self.source_info = file_util.FileHandle.Get(url).Info()
return
# Corresponding build channel found, use its download context
self.source_url = build_item_path
self.download_fn = build_channel.DownloadFile
build_item = build_channel.GetBuildItem(build_item_path)
if build_item:
self.source_info = file_util.FileInfo(
url=self.url,
total_size=build_item.size,
timestamp=build_item.timestamp)
def testLookupBuildItem_withHttpUrl(self, mock_find_handle_get):
url = 'http://foo.com/bar/zzz/file.ext?foo=bar'
mock_file_handle = mock.MagicMock()
mock_find_handle_get.return_value = mock_file_handle
mock_file_info = file_util.FileInfo(
url=url,
is_file=True,
total_size=1234,
timestamp=datetime.datetime.utcnow())
mock_file_handle.Info.return_value = mock_file_info
res = self.app.get(
'/_ah/api/mtt/v1/build_channels/build_item_lookup?url=%s' %
(urllib.parse.quote(url)))
build_item_msg = protojson.decode_message(messages.BuildItem, res.body)
mock_find_handle_get.assert_called_once_with(url)
mock_file_handle.Info.assert_called_once()
self.assertEqual('file.ext', build_item_msg.name)
self.assertEqual(mock_file_info.is_file, build_item_msg.is_file)
self.assertEqual(mock_file_info.total_size, build_item_msg.size)
self.assertEqual(mock_file_info.timestamp, build_item_msg.timestamp)
def Info(self):
return file_util.FileInfo(url=self.url,
total_size=len(self.value),
content_type='string')