class FileSystemTest(unittest.TestCase):
path = '/tmp/luigi-test-dir'
fs = LocalFileSystem()
def setUp(self):
if os.path.exists(self.path):
shutil.rmtree(self.path)
def tearDown(self):
self.setUp()
def test_copy(self):
src = os.path.join(self.path, 'src.txt')
dest = os.path.join(self.path, 'newdir', 'dest.txt')
LocalTarget(src).open('w').close()
self.fs.copy(src, dest)
self.assertTrue(os.path.exists(src))
self.assertTrue(os.path.exists(dest))
def test_mkdir(self):
testpath = os.path.join(self.path, 'foo/bar')
self.assertRaises(MissingParentDirectory,
self.fs.mkdir,
testpath,
parents=False)
self.fs.mkdir(testpath)
self.assertTrue(os.path.exists(testpath))
self.assertTrue(self.fs.isdir(testpath))
self.assertRaises(FileAlreadyExists,
self.fs.mkdir,
testpath,
raise_if_exists=True)
def test_exists(self):
self.assertFalse(self.fs.exists(self.path))
os.mkdir(self.path)
self.assertTrue(self.fs.exists(self.path))
self.assertTrue(self.fs.isdir(self.path))
def test_listdir(self):
os.mkdir(self.path)
with open(self.path + '/file', 'w'):
pass
self.assertTrue([self.path + '/file'],
list(self.fs.listdir(self.path + '/')))
def test_move_to_new_dir(self):
# Regression test for a bug in LocalFileSystem.move
src = os.path.join(self.path, 'src.txt')
dest = os.path.join(self.path, 'newdir', 'dest.txt')
LocalTarget(src).open('w').close()
self.fs.move(src, dest)
self.assertTrue(os.path.exists(dest))
def run(self):
local_fs = LocalFileSystem()
local_fs.mkdir(self.output().path)
def after_run(self):
# delete temp dir
filesys = LocalFileSystem()
if self.mount_tmp and filesys.exists(self._host_tmp_dir):
filesys.remove(self._host_tmp_dir, recursive=True)
def run(self):
# get image if missing
if self.force_pull or len(self._client.images(name=self._image)) == 0:
logger.info('Pulling docker image ' + self._image)
try:
for logline in self._client.pull(self._image, stream=True):
logger.debug(logline.decode('utf-8'))
except APIError as e:
self.__logger.warning("Error in Docker API: " + e.explanation)
raise
# remove clashing container if a container with the same name exists
if self.auto_remove and self.name:
try:
self._client.remove_container(self.name, force=True)
except APIError as e:
self.__logger.warning("Ignored error in Docker API: " +
e.explanation)
# run the container
try:
logger.debug('Creating image: %s command: %s volumes: %s' %
(self._image, self.command, self._binds))
host_config = self._client.create_host_config(
binds=self._binds, network_mode=self.network_mode)
container = self._client.create_container(
self._image,
command=self.command,
name=self.name,
environment=self.environment,
volumes=self._volumes,
host_config=host_config,
**self.container_options)
self._client.start(container['Id'])
exit_status = self._client.wait(container['Id'])
# docker-py>=3.0.0 returns a dict instead of the status code directly
if type(exit_status) is dict:
exit_status = exit_status['StatusCode']
if exit_status != 0:
stdout = False
stderr = True
error = self._client.logs(container['Id'],
stdout=stdout,
stderr=stderr)
if self.auto_remove:
try:
self._client.remove_container(container['Id'])
except docker.errors.APIError:
self.__logger.warning("Container " + container['Id'] +
" could not be removed")
if exit_status != 0:
raise ContainerError(container, exit_status, self.command,
self._image, error)
except ContainerError as e:
# catch non zero exti status and return it
container_name = ''
if self.name:
container_name = self.name
try:
message = e.message
except AttributeError:
message = str(e)
self.__logger.error("Container " + container_name +
" exited with non zero code: " + message)
raise
except ImageNotFound:
self.__logger.error("Image " + self._image + " not found")
raise
except APIError as e:
self.__logger.error("Error in Docker API: " + e.explanation)
raise
# delete temp dir
filesys = LocalFileSystem()
if self.mount_tmp and filesys.exists(self._host_tmp_dir):
filesys.remove(self._host_tmp_dir, recursive=True)
def run(self):
local_fs = LocalFileSystem()
for target in self.output():
local_fs.mkdir(target.path)
def run(self):
# get image if missing
if self.force_pull or len(self._client.images(name=self._image)) == 0:
logger.info('Pulling docker image ' + self._image)
try:
for logline in self._client.pull(self._image, stream=True):
logger.debug(logline.decode('utf-8'))
except APIError as e:
self.__logger.warning("Error in Docker API: " + e.explanation)
raise
# remove clashing container if a container with the same name exists
if self.auto_remove and self.name:
try:
self._client.remove_container(self.name,
force=True)
except APIError as e:
self.__logger.warning("Ignored error in Docker API: " + e.explanation)
# run the container
try:
logger.debug('Creating image: %s command: %s volumes: %s'
% (self._image, self.command, self._binds))
host_config = self._client.create_host_config(binds=self._binds,
network_mode=self.network_mode)
container = self._client.create_container(self._image,
command=self.command,
name=self.name,
environment=self.environment,
volumes=self._volumes,
host_config=host_config,
**self.container_options)
self._client.start(container['Id'])
exit_status = self._client.wait(container['Id'])
# docker-py>=3.0.0 returns a dict instead of the status code directly
if type(exit_status) is dict:
exit_status = exit_status['StatusCode']
if exit_status != 0:
stdout = False
stderr = True
error = self._client.logs(container['Id'],
stdout=stdout,
stderr=stderr)
if self.auto_remove:
try:
self._client.remove_container(container['Id'])
except docker.errors.APIError:
self.__logger.warning("Container " + container['Id'] +
" could not be removed")
if exit_status != 0:
raise ContainerError(container, exit_status, self.command, self._image, error)
except ContainerError as e:
# catch non zero exti status and return it
container_name = ''
if self.name:
container_name = self.name
try:
message = e.message
except AttributeError:
message = str(e)
self.__logger.error("Container " + container_name +
" exited with non zero code: " + message)
raise
except ImageNotFound as e:
self.__logger.error("Image " + self._image + " not found")
raise
except APIError as e:
self.__logger.error("Error in Docker API: "+e.explanation)
raise
# delete temp dir
filesys = LocalFileSystem()
if self.mount_tmp and filesys.exists(self._host_tmp_dir):
filesys.remove(self._host_tmp_dir, recursive=True)