def _test(self,
module_name,
shouldBelongToToil=False,
expectedContents=None):
module = ModuleDescriptor.forModule(module_name)
# Assert basic attributes and properties
self.assertEqual(module.belongsToToil, shouldBelongToToil)
self.assertEquals(module.name, module_name)
if shouldBelongToToil:
self.assertTrue(module.dirPath.endswith('/src'))
# Before the module is saved as a resource, localize() and globalize() are identity
# methods. This should log warnings.
self.assertIs(module.localize(), module)
self.assertIs(module.globalize(), module)
# Create a mock job store ...
jobStore = MagicMock()
# ... to generate a fake URL for the resource ...
url = 'file://foo.zip'
jobStore.getSharedPublicUrl.return_value = url
# ... and save the resource to it.
resource = module.saveAsResourceTo(jobStore)
# Ensure that the URL generation method is actually called, ...
jobStore.getSharedPublicUrl.assert_called_once_with(resource.pathHash)
# ... and that ensure that writeSharedFileStream is called.
jobStore.writeSharedFileStream.assert_called_once_with(
resource.pathHash, isProtected=False)
# Now it gets a bit complicated: Ensure that the context manager returned by the
# jobStore's writeSharedFileStream() method is entered and that the file handle yielded
# by the context manager is written to once with the zipped source tree from which
# 'toil.resource' was orginally imported. Keep the zipped tree around such that we can
# mock the download later.
file_handle = jobStore.writeSharedFileStream.return_value.__enter__.return_value
# The first 0 index selects the first call of write(), the second 0 selects positional
# instead of keyword arguments, and the third 0 selects the first positional, i.e. the
# contents. This is a bit brittle since it assumes that all the data is written in a
# single call to write(). If more calls are made we can easily concatenate them.
zipFile = file_handle.write.call_args_list[0][0][0]
self.assertTrue(
zipFile.startswith('PK')) # the magic header for ZIP files
# Check contents if requested
if expectedContents is not None:
with ZipFile(BytesIO(zipFile)) as _zipFile:
self.assertEqual(set(_zipFile.namelist()), expectedContents)
self.assertEquals(resource.url, url)
# Now we're on the worker. Prepare the storage for localized resources
Resource.prepareSystem()
# Register the resource for subsequent lookup.
resource.register()
# Lookup the resource and ensure that the result is equal to but not the same as the
# original resource. Lookup will also be used when we localize the module that was
# originally used to create the resource.
localResource = Resource.lookup(module._resourcePath)
self.assertEquals(resource, localResource)
self.assertIsNot(resource, localResource)
# Now show that we can localize the module using the registered resource. Set up a mock
# urlopen() that yields the zipped tree ...
mock_urlopen = MagicMock()
mock_urlopen.return_value.read.return_value = zipFile
with patch('toil.resource.urlopen', mock_urlopen):
# ... and use it to download and unpack the resource
localModule = module.localize()
# The name should be equal between original and localized resource ...
self.assertEquals(module.name, localModule.name)
# ... but the directory should be different.
self.assertNotEquals(module.dirPath, localModule.dirPath)
# Show that we can 'undo' localization. This is necessary when the user script's jobs are
# invoked on the worker where they generate more child jobs.
self.assertEquals(localModule.globalize(), module)
def _test(self, module_name,
shouldBelongToToil=False, expectedContents=None, allowExtraContents=True):
module = ModuleDescriptor.forModule(module_name)
# Assert basic attributes and properties
self.assertEqual(module.belongsToToil, shouldBelongToToil)
self.assertEquals(module.name, module_name)
if shouldBelongToToil:
self.assertTrue(module.dirPath.endswith('/src'))
# Before the module is saved as a resource, localize() and globalize() are identity
# methods. This should log warnings.
self.assertIs(module.localize(), module)
self.assertIs(module.globalize(), module)
# Create a mock job store ...
jobStore = MagicMock()
# ... to generate a fake URL for the resource ...
url = 'file://foo.zip'
jobStore.getSharedPublicUrl.return_value = url
# ... and save the resource to it.
resource = module.saveAsResourceTo(jobStore)
# Ensure that the URL generation method is actually called, ...
jobStore.getSharedPublicUrl.assert_called_once_with(resource.pathHash)
# ... and that ensure that writeSharedFileStream is called.
jobStore.writeSharedFileStream.assert_called_once_with(resource.pathHash,
isProtected=False)
# Now it gets a bit complicated: Ensure that the context manager returned by the
# jobStore's writeSharedFileStream() method is entered and that the file handle yielded
# by the context manager is written to once with the zipped source tree from which
# 'toil.resource' was orginally imported. Keep the zipped tree around such that we can
# mock the download later.
file_handle = jobStore.writeSharedFileStream.return_value.__enter__.return_value
# The first 0 index selects the first call of write(), the second 0 selects positional
# instead of keyword arguments, and the third 0 selects the first positional, i.e. the
# contents. This is a bit brittle since it assumes that all the data is written in a
# single call to write(). If more calls are made we can easily concatenate them.
zipFile = file_handle.write.call_args_list[0][0][0]
self.assertTrue(zipFile.startswith('PK')) # the magic header for ZIP files
# Check contents if requested
if expectedContents is not None:
with ZipFile(BytesIO(zipFile)) as _zipFile:
actualContents = set(_zipFile.namelist())
if allowExtraContents:
self.assertTrue(actualContents.issuperset(expectedContents))
else:
self.assertEqual(actualContents, expectedContents)
self.assertEquals(resource.url, url)
# Now we're on the worker. Prepare the storage for localized resources
Resource.prepareSystem()
try:
# Register the resource for subsequent lookup.
resource.register()
# Lookup the resource and ensure that the result is equal to but not the same as the
# original resource. Lookup will also be used when we localize the module that was
# originally used to create the resource.
localResource = Resource.lookup(module._resourcePath)
self.assertEquals(resource, localResource)
self.assertIsNot(resource, localResource)
# Now show that we can localize the module using the registered resource. Set up a mock
# urlopen() that yields the zipped tree ...
mock_urlopen = MagicMock()
mock_urlopen.return_value.read.return_value = zipFile
with patch('toil.resource.urlopen', mock_urlopen):
# ... and use it to download and unpack the resource
localModule = module.localize()
# The name should be equal between original and localized resource ...
self.assertEquals(module.name, localModule.name)
# ... but the directory should be different.
self.assertNotEquals(module.dirPath, localModule.dirPath)
# Show that we can 'undo' localization. This is necessary when the user script's jobs
# are invoked on the worker where they generate more child jobs.
self.assertEquals(localModule.globalize(), module)
finally:
Resource.cleanSystem()
