def download_files(self, volume_path, file_downloads):
"""See :meth:`storage.brokers.broker.Broker.download_files`"""
with S3Client(self._credentials, self._region_name) as client:
for file_download in file_downloads:
# If file supports partial mount and volume is configured attempt sym-link
if file_download.partial and self._volume:
logger.debug('Partial S3 file accessed by mounted bucket.')
path_to_download = os.path.join(
volume_path, file_download.file.file_path)
logger.info('Checking path %s', path_to_download)
if not os.path.exists(path_to_download):
raise MissingFile(file_download.file.file_name)
# Create symlink to the file in the host mount
logger.info('Creating link %s -> %s',
file_download.local_path, path_to_download)
execute_command_line([
'ln', '-s', path_to_download, file_download.local_path
])
# Fall-back to default S3 file download
else:
try:
s3_object = client.get_object(
self._bucket_name, file_download.file.file_path)
except FileDoesNotExist:
raise MissingFile(file_download.file.file_name)
self._download_file(s3_object, file_download.file,
file_download.local_path)
def move_files(self, volume_path, file_moves):
"""See :meth:`storage.brokers.broker.Broker.move_files`
"""
for file_move in file_moves:
full_old_path = os.path.join(volume_path, file_move.file.file_path)
full_new_path = os.path.join(volume_path, file_move.new_path)
full_new_path_dir = os.path.dirname(full_new_path)
logger.info('Checking path %s', full_old_path)
if not os.path.exists(full_old_path):
raise MissingFile(file_move.file.file_name)
if not os.path.exists(full_new_path_dir):
logger.info('Creating %s', full_new_path_dir)
os.makedirs(full_new_path_dir, mode=0755)
logger.info('Moving %s to %s', full_old_path, full_new_path)
shutil.move(full_old_path, full_new_path)
logger.info('Setting file permissions for %s', full_new_path)
os.chmod(full_new_path, 0644)
# Update model attributes
file_move.file.file_path = file_move.new_path
file_move.file.save()
def ready(self):
"""Registers components related to storage"""
import storage.brokers.factory as factory
from storage.brokers.host_broker import HostBroker
from storage.brokers.nfs_broker import NfsBroker
from storage.brokers.s3_broker import S3Broker
# Register broker types
factory.add_broker_type(HostBroker)
factory.add_broker_type(NfsBroker)
factory.add_broker_type(S3Broker)
# Register storage errors
from error.exceptions import register_error
from storage.exceptions import DeletedFile, MissingFile
register_error(DeletedFile(''))
register_error(MissingFile(''))
# Register storage message types
from storage.messages.delete_files import DeleteFiles
from storage.messages.move_files import MoveFile
from messaging.messages.factory import add_message_type
add_message_type(DeleteFiles)
add_message_type(MoveFile)
def download_files(self, volume_path, file_downloads):
"""See :meth:`storage.brokers.broker.Broker.download_files`
"""
for file_download in file_downloads:
path_to_download = os.path.join(volume_path, file_download.file.file_path)
logger.info('Checking path %s', path_to_download)
if not os.path.exists(path_to_download):
raise MissingFile(file_download.file.file_name)
# Create symlink to the file in the host mount
logger.info('Creating link %s -> %s', file_download.local_path, path_to_download)
execute_command_line(['ln', '-s', path_to_download, file_download.local_path])
def move_files(self, volume_path, file_moves):
"""See :meth:`storage.brokers.broker.Broker.move_files`"""
with S3Client(self._credentials, self._region_name) as client:
for file_move in file_moves:
try:
s3_object_src = client.get_object(self._bucket_name, file_move.file.file_path)
except FileDoesNotExist:
raise MissingFile(file_move.file.file_name)
s3_object_dest = client.get_object(self._bucket_name, file_move.new_path, False)
self._move_file(s3_object_src, s3_object_dest, file_move.file, file_move.new_path)
# Update model attributes
file_move.file.file_path = file_move.new_path
file_move.file.save()
def ready(self):
"""Registers the storage brokers and storage errors"""
import storage.brokers.factory as factory
from storage.brokers.host_broker import HostBroker
from storage.brokers.nfs_broker import NfsBroker
from storage.brokers.s3_broker import S3Broker
# Register broker types
factory.add_broker_type(HostBroker)
factory.add_broker_type(NfsBroker)
factory.add_broker_type(S3Broker)
# Register storage errors
from error.exceptions import register_error
from storage.exceptions import DeletedFile, MissingFile
register_error(DeletedFile(''))
register_error(MissingFile(''))