def test_put_file(self, _mkdir, shutil):
path_in = '/tmp/test'
unit = Mock(id='123', storage_path='/tmp/storage')
storage = FileStorage()
# test
storage.put(unit, path_in)
# validation
_mkdir.assert_called_once_with(os.path.dirname(unit.storage_path))
shutil.copy.assert_called_once_with(path_in, unit.storage_path)
def test_put_file(self, _mkdir, shutil):
path_in = '/tmp/test'
unit = Mock(id='123', storage_path='/tmp/storage')
storage = FileStorage()
# test
storage.put(unit, path_in)
# validation
_mkdir.assert_called_once_with(os.path.dirname(unit.storage_path))
shutil.copy.assert_called_once_with(path_in, unit.storage_path)
def test_put_file_with_location(self, _mkdir, shutil):
path_in = '/tmp/test'
location = '/a/b/'
unit = Mock(id='123', storage_path='/tmp/storage')
storage = FileStorage()
# test
storage.put(unit, path_in, location)
# validation
destination = os.path.join(unit.storage_path, location.lstrip('/'))
_mkdir.assert_called_once_with(os.path.dirname(destination))
shutil.copy.assert_called_once_with(path_in, destination)
def test_put_file_with_location(self, _mkdir, shutil):
path_in = '/tmp/test'
location = '/a/b/'
unit = Mock(id='123', storage_path='/tmp/storage')
storage = FileStorage()
# test
storage.put(unit, path_in, location)
# validation
destination = os.path.join(unit.storage_path, location.lstrip('/'))
_mkdir.assert_called_once_with(os.path.dirname(destination))
shutil.copy.assert_called_once_with(path_in, destination)
def test_put_file_with_location(self, _mkdir, shutil, tempfile, close, rename):
path_in = '/tmp/test'
location = '/a/b/'
temp_destination = '/some/file/path'
unit = Mock(id='123', storage_path='/tmp/storage')
storage = FileStorage()
tempfile.mkstemp.return_value = ('fd', temp_destination)
# test
storage.put(unit, path_in, location)
# validation
destination = os.path.join(unit.storage_path, location.lstrip('/'))
_mkdir.assert_called_once_with(os.path.dirname(destination))
shutil.copy.assert_called_once_with(path_in, temp_destination)
rename.assert_called_once_with(temp_destination, destination)
def import_content(self, path, location=None):
"""
Import a content file into platform storage.
The (optional) *location* may be used to specify a path within the unit
storage where the content is to be stored.
For example:
import_content('/tmp/file') will store 'file' at: _storage_path
import_content('/tmp/file', 'a/b/c) will store 'file' at: _storage_path/a/b/c
:param path: The absolute path to the file to be imported.
:type path: str
:param location: The (optional) location within the unit storage path
where the content is to be stored.
:type location: str
:raises ImportError: if the unit has not been saved.
:raises PulpCodedException: PLP0037 if *path* is not an existing file.
"""
if not self._last_updated:
raise ImportError("Content unit must be saved before associated content"
" files can be imported.")
if not os.path.isfile(path):
raise exceptions.PulpCodedException(error_code=error_codes.PLP0037, path=path)
with FileStorage() as storage:
storage.put(self, path, location)
def test_put_file(self, config, shutil):
path_in = '/tmp/test'
storage_dir = '/tmp/storage'
unit = Mock(id='0123456789', unit_type_id='ABC')
config.get = lambda s, p: {'server': {'storage_dir': storage_dir}}[s][p]
storage = FileStorage()
# test
storage.put(unit, path_in)
# validation
destination = os.path.join(
os.path.join(storage_dir, 'content', 'units', unit.unit_type_id),
unit.id[0:4], unit.id)
shutil.copy.assert_called_once_with(path_in, destination)
self.assertEqual(unit.storage_path, destination)
def test_put_file_with_location(self, _mkdir, shutil, tempfile, close,
rename):
path_in = '/tmp/test'
location = '/a/b/'
temp_destination = '/some/file/path'
unit = Mock(id='123', storage_path='/tmp/storage')
storage = FileStorage()
tempfile.mkstemp.return_value = ('fd', temp_destination)
# test
storage.put(unit, path_in, location)
# validation
destination = os.path.join(unit.storage_path, location.lstrip('/'))
_mkdir.assert_called_once_with(os.path.dirname(destination))
shutil.copy.assert_called_once_with(path_in, temp_destination)
rename.assert_called_once_with(temp_destination, destination)
def test_put_file_correct_size(self, _mkdir, shutil, tempfile, close, rename):
path_in = '/tmp/test'
temp_destination = '/some/file/path'
unit = Mock(id='123', storage_path='/tmp/storage')
storage = FileStorage()
tempfile.mkstemp.return_value = ('fd', temp_destination)
# test
storage.put(unit, path_in)
# validation
_mkdir.assert_called_once_with(os.path.dirname(unit.storage_path))
tempfile.mkstemp.assert_called_once_with(dir=os.path.dirname(unit.storage_path))
close.assert_called_once_with('fd')
shutil.copy.assert_called_once_with(path_in, temp_destination)
unit.verify_size.assert_called_once_with(temp_destination)
rename.assert_called_once_with(temp_destination, unit.storage_path)
def test_put_file_no_verify_size(self, _mkdir, shutil, tempfile, close, remove, rename):
path_in = '/tmp/test'
temp_destination = '/some/file/path'
unit = Mock(id='123', storage_path='/tmp/storage')
storage = FileStorage()
tempfile.mkstemp.return_value = ('fd', temp_destination)
unit.verify_size.side_effect = AttributeError("object has no attribute 'verify_size'")
# test
storage.put(unit, path_in)
# validation
_mkdir.assert_called_once_with(os.path.dirname(unit.storage_path))
tempfile.mkstemp.assert_called_once_with(dir=os.path.dirname(unit.storage_path))
close.assert_called_once_with('fd')
shutil.copy.assert_called_once_with(path_in, temp_destination)
unit.verify_size.assert_called_once_with(temp_destination)
self.assertFalse(remove.called)
rename.assert_called_once_with(temp_destination, unit.storage_path)
def test_put_file_correct_size(self, _mkdir, shutil, tempfile, close,
rename):
path_in = '/tmp/test'
temp_destination = '/some/file/path'
unit = Mock(id='123', storage_path='/tmp/storage')
storage = FileStorage()
tempfile.mkstemp.return_value = ('fd', temp_destination)
# test
storage.put(unit, path_in)
# validation
_mkdir.assert_called_once_with(os.path.dirname(unit.storage_path))
tempfile.mkstemp.assert_called_once_with(
dir=os.path.dirname(unit.storage_path))
close.assert_called_once_with('fd')
shutil.copy.assert_called_once_with(path_in, temp_destination)
unit.verify_size.assert_called_once_with(temp_destination)
rename.assert_called_once_with(temp_destination, unit.storage_path)
def test_put_file_no_verify_size(self, _mkdir, shutil, tempfile, close,
remove, rename):
path_in = '/tmp/test'
temp_destination = '/some/file/path'
unit = Mock(id='123', storage_path='/tmp/storage')
storage = FileStorage()
tempfile.mkstemp.return_value = ('fd', temp_destination)
unit.verify_size.side_effect = AttributeError(
"object has no attribute 'verify_size'")
# test
storage.put(unit, path_in)
# validation
_mkdir.assert_called_once_with(os.path.dirname(unit.storage_path))
tempfile.mkstemp.assert_called_once_with(
dir=os.path.dirname(unit.storage_path))
close.assert_called_once_with('fd')
shutil.copy.assert_called_once_with(path_in, temp_destination)
unit.verify_size.assert_called_once_with(temp_destination)
self.assertFalse(remove.called)
rename.assert_called_once_with(temp_destination, unit.storage_path)
def test_get_path(self, config):
storage_dir = '/tmp/storage'
config.get = lambda s, p: {'server': {'storage_dir': storage_dir}}[s][p]
unit = Mock(id='0123456789', type_id='ABC')
# test
path = FileStorage.get_path(unit)
# validation
self.assertEqual(
path,
os.path.join(storage_dir, 'content', 'units',
unit.type_id,
unit.id[0:4],
unit.id))
def set_storage_path(self, filename=None):
"""
Set the storage path.
This is a total hack to support existing single-file units with a
_storage_path that includes the file name.
:param filename: An optional filename to appended to the path.
:rtype filename: str
"""
path = FileStorage.get_path(self)
if filename:
if not os.path.isabs(filename):
path = os.path.join(path, filename)
else:
raise ValueError(_('must be relative path'))
self._storage_path = path
def set_storage_path(self, filename=None):
"""
Set the storage path.
This is a total hack to support existing single-file units with a
_storage_path that includes the file name.
:param filename: An optional filename to appended to the path.
:rtype filename: str
"""
path = FileStorage.get_path(self)
if filename:
if not os.path.isabs(filename):
path = os.path.join(path, filename)
else:
raise ValueError(_('must be relative path'))
self._storage_path = path
def pre_save_signal(cls, sender, document, **kwargs):
"""
The signal that is triggered before a unit is saved, this is used to
support the legacy behavior of generating the unit id and setting
the last_updated timestamp
:param sender: sender class
:type sender: object
:param document: Document that sent the signal
:type document: FileContentUnit
"""
super(FileContentUnit, cls).pre_save_signal(sender, document, **kwargs)
if not document._source_location:
# no content
return
with FileStorage() as storage:
storage.put(document, document._source_location)
def test_get_path(self, config):
storage_dir = '/tmp/storage'
config.get = lambda s, p: {
'server': {
'storage_dir': storage_dir
}
}[s][p]
unit = Mock(id='0123456789', type_id='ABC')
# test
path = FileStorage.get_path(unit)
# validation
self.assertEqual(
path,
os.path.join(storage_dir, 'content', 'units', unit.type_id,
unit.id[0:4], unit.id))
def test_get_path(self, config, sha256):
storage_dir = '/tmp/storage'
digest = '0123456789'
config.get = lambda s, p: {'server': {'storage_dir': storage_dir}}[s][p]
unit = Mock(type_id='ABC')
unit.unit_key_as_digest.return_value = digest
# test
path = FileStorage.get_path(unit)
# validation
unit.unit_key_as_digest.assert_called_once_with(sha256.return_value)
self.assertEqual(
path,
os.path.join(storage_dir, 'content', 'units',
unit.type_id,
digest[0:2],
digest[2:]))
def download_succeeded(self, report):
"""
Marks the individual file for the unit as downloaded and moves it into
its final storage location if its checksum value matches the value in
the catalog entry (if present).
Inherited from DownloadEventListener.
:param report: the report associated with the download request.
:type report: nectar.report.DownloadReport
"""
# Reload the content unit
unit_model = plugin_api.get_unit_model_by_id(report.data[TYPE_ID])
unit_qs = unit_model.objects.filter(id=report.data[UNIT_ID])
content_unit = unit_qs.only('_content_type_id', 'id',
'_last_updated').get()
path_entry = report.data[UNIT_FILES][report.destination]
# Validate the file and update the progress.
catalog_entry = path_entry[CATALOG_ENTRY]
try:
self.validate_file(report.destination,
catalog_entry.checksum_algorithm,
catalog_entry.checksum)
relative_path = os.path.relpath(catalog_entry.path,
FileStorage.get_path(content_unit))
if len(report.data[UNIT_FILES]) == 1:
# If the unit is single-file, update the storage path to point to the file
content_unit.set_storage_path(relative_path)
unit_qs.update_one(
set___storage_path=content_unit._storage_path)
content_unit.import_content(report.destination)
else:
content_unit.import_content(report.destination,
location=relative_path)
self.progress_successes += 1
path_entry[PATH_DOWNLOADED] = True
except (InvalidChecksumType, VerificationException, IOError), e:
_logger.debug(
_('Download of {path} failed: {reason}.').format(
path=catalog_entry.path, reason=str(e)))
path_entry[PATH_DOWNLOADED] = False
self.progress_failures += 1
def download_succeeded(self, report):
"""
Marks the individual file for the unit as downloaded and moves it into
its final storage location if its checksum value matches the value in
the catalog entry (if present).
Inherited from DownloadEventListener.
:param report: the report associated with the download request.
:type report: nectar.report.DownloadReport
"""
# Reload the content unit
unit_model = plugin_api.get_unit_model_by_id(report.data[TYPE_ID])
unit_qs = unit_model.objects.filter(id=report.data[UNIT_ID])
content_unit = unit_qs.only('_content_type_id', 'id', '_last_updated').get()
path_entry = report.data[UNIT_FILES][report.destination]
# Validate the file and update the progress.
catalog_entry = path_entry[CATALOG_ENTRY]
try:
self.validate_file(
report.destination,
catalog_entry.checksum_algorithm,
catalog_entry.checksum
)
relative_path = os.path.relpath(
catalog_entry.path,
FileStorage.get_path(content_unit)
)
if len(report.data[UNIT_FILES]) == 1:
# If the unit is single-file, update the storage path to point to the file
content_unit.set_storage_path(relative_path)
unit_qs.update_one(set___storage_path=content_unit._storage_path)
content_unit.import_content(report.destination)
else:
content_unit.import_content(report.destination, location=relative_path)
self.progress_successes += 1
path_entry[PATH_DOWNLOADED] = True
except (InvalidChecksumType, VerificationException, IOError), e:
_logger.debug(_('Download of {path} failed: {reason}.').format(
path=catalog_entry.path, reason=str(e)))
path_entry[PATH_DOWNLOADED] = False
self.progress_failures += 1
def test_get_path(self, config, sha256):
storage_dir = '/tmp/storage'
digest = '0123456789'
config.get = lambda s, p: {
'server': {
'storage_dir': storage_dir
}
}[s][p]
unit = Mock(type_id='ABC')
unit.unit_key_as_digest.return_value = digest
# test
path = FileStorage.get_path(unit)
# validation
unit.unit_key_as_digest.assert_called_once_with(sha256.return_value)
self.assertEqual(
path,
os.path.join(storage_dir, 'content', 'units', unit.type_id,
digest[0:2], digest[2:]))
def test_put_file_incorrect_size(self, _mkdir, shutil, tempfile, close,
remove, rename):
path_in = '/tmp/test'
temp_destination = '/some/file/path'
unit = Mock(id='123', storage_path='/tmp/storage')
storage = FileStorage()
tempfile.mkstemp.return_value = ('fd', temp_destination)
unit.verify_size.side_effect = verification.VerificationException(22)
# test
self.assertRaises(verification.VerificationException, storage.put,
unit, path_in)
# validation
_mkdir.assert_called_once_with(os.path.dirname(unit.storage_path))
tempfile.mkstemp.assert_called_once_with(
dir=os.path.dirname(unit.storage_path))
close.assert_called_once_with('fd')
shutil.copy.assert_called_once_with(path_in, temp_destination)
unit.verify_size.assert_called_once_with(temp_destination)
remove.assert_called_once_with(temp_destination)
self.assertFalse(rename.called)
def test_get(self):
storage = FileStorage()
storage.get(None) # just for coverage
