def test_put_file(self, _mkdir, shutil): path_in = '/tmp/test' unit = Mock(id='123', storage_path='/tmp/storage') storage = FileStorage() # test storage.put(unit, path_in) # validation _mkdir.assert_called_once_with(os.path.dirname(unit.storage_path)) shutil.copy.assert_called_once_with(path_in, unit.storage_path)
def test_put_file_with_location(self, _mkdir, shutil): path_in = '/tmp/test' location = '/a/b/' unit = Mock(id='123', storage_path='/tmp/storage') storage = FileStorage() # test storage.put(unit, path_in, location) # validation destination = os.path.join(unit.storage_path, location.lstrip('/')) _mkdir.assert_called_once_with(os.path.dirname(destination)) shutil.copy.assert_called_once_with(path_in, destination)
def test_put_file_with_location(self, _mkdir, shutil, tempfile, close, rename): path_in = '/tmp/test' location = '/a/b/' temp_destination = '/some/file/path' unit = Mock(id='123', storage_path='/tmp/storage') storage = FileStorage() tempfile.mkstemp.return_value = ('fd', temp_destination) # test storage.put(unit, path_in, location) # validation destination = os.path.join(unit.storage_path, location.lstrip('/')) _mkdir.assert_called_once_with(os.path.dirname(destination)) shutil.copy.assert_called_once_with(path_in, temp_destination) rename.assert_called_once_with(temp_destination, destination)
def import_content(self, path, location=None): """ Import a content file into platform storage. The (optional) *location* may be used to specify a path within the unit storage where the content is to be stored. For example: import_content('/tmp/file') will store 'file' at: _storage_path import_content('/tmp/file', 'a/b/c) will store 'file' at: _storage_path/a/b/c :param path: The absolute path to the file to be imported. :type path: str :param location: The (optional) location within the unit storage path where the content is to be stored. :type location: str :raises ImportError: if the unit has not been saved. :raises PulpCodedException: PLP0037 if *path* is not an existing file. """ if not self._last_updated: raise ImportError("Content unit must be saved before associated content" " files can be imported.") if not os.path.isfile(path): raise exceptions.PulpCodedException(error_code=error_codes.PLP0037, path=path) with FileStorage() as storage: storage.put(self, path, location)
def test_put_file(self, config, shutil): path_in = '/tmp/test' storage_dir = '/tmp/storage' unit = Mock(id='0123456789', unit_type_id='ABC') config.get = lambda s, p: {'server': {'storage_dir': storage_dir}}[s][p] storage = FileStorage() # test storage.put(unit, path_in) # validation destination = os.path.join( os.path.join(storage_dir, 'content', 'units', unit.unit_type_id), unit.id[0:4], unit.id) shutil.copy.assert_called_once_with(path_in, destination) self.assertEqual(unit.storage_path, destination)
def test_put_file_correct_size(self, _mkdir, shutil, tempfile, close, rename): path_in = '/tmp/test' temp_destination = '/some/file/path' unit = Mock(id='123', storage_path='/tmp/storage') storage = FileStorage() tempfile.mkstemp.return_value = ('fd', temp_destination) # test storage.put(unit, path_in) # validation _mkdir.assert_called_once_with(os.path.dirname(unit.storage_path)) tempfile.mkstemp.assert_called_once_with(dir=os.path.dirname(unit.storage_path)) close.assert_called_once_with('fd') shutil.copy.assert_called_once_with(path_in, temp_destination) unit.verify_size.assert_called_once_with(temp_destination) rename.assert_called_once_with(temp_destination, unit.storage_path)
def test_put_file_no_verify_size(self, _mkdir, shutil, tempfile, close, remove, rename): path_in = '/tmp/test' temp_destination = '/some/file/path' unit = Mock(id='123', storage_path='/tmp/storage') storage = FileStorage() tempfile.mkstemp.return_value = ('fd', temp_destination) unit.verify_size.side_effect = AttributeError("object has no attribute 'verify_size'") # test storage.put(unit, path_in) # validation _mkdir.assert_called_once_with(os.path.dirname(unit.storage_path)) tempfile.mkstemp.assert_called_once_with(dir=os.path.dirname(unit.storage_path)) close.assert_called_once_with('fd') shutil.copy.assert_called_once_with(path_in, temp_destination) unit.verify_size.assert_called_once_with(temp_destination) self.assertFalse(remove.called) rename.assert_called_once_with(temp_destination, unit.storage_path)
def test_put_file_correct_size(self, _mkdir, shutil, tempfile, close, rename): path_in = '/tmp/test' temp_destination = '/some/file/path' unit = Mock(id='123', storage_path='/tmp/storage') storage = FileStorage() tempfile.mkstemp.return_value = ('fd', temp_destination) # test storage.put(unit, path_in) # validation _mkdir.assert_called_once_with(os.path.dirname(unit.storage_path)) tempfile.mkstemp.assert_called_once_with( dir=os.path.dirname(unit.storage_path)) close.assert_called_once_with('fd') shutil.copy.assert_called_once_with(path_in, temp_destination) unit.verify_size.assert_called_once_with(temp_destination) rename.assert_called_once_with(temp_destination, unit.storage_path)
def test_put_file_no_verify_size(self, _mkdir, shutil, tempfile, close, remove, rename): path_in = '/tmp/test' temp_destination = '/some/file/path' unit = Mock(id='123', storage_path='/tmp/storage') storage = FileStorage() tempfile.mkstemp.return_value = ('fd', temp_destination) unit.verify_size.side_effect = AttributeError( "object has no attribute 'verify_size'") # test storage.put(unit, path_in) # validation _mkdir.assert_called_once_with(os.path.dirname(unit.storage_path)) tempfile.mkstemp.assert_called_once_with( dir=os.path.dirname(unit.storage_path)) close.assert_called_once_with('fd') shutil.copy.assert_called_once_with(path_in, temp_destination) unit.verify_size.assert_called_once_with(temp_destination) self.assertFalse(remove.called) rename.assert_called_once_with(temp_destination, unit.storage_path)
def test_get_path(self, config): storage_dir = '/tmp/storage' config.get = lambda s, p: {'server': {'storage_dir': storage_dir}}[s][p] unit = Mock(id='0123456789', type_id='ABC') # test path = FileStorage.get_path(unit) # validation self.assertEqual( path, os.path.join(storage_dir, 'content', 'units', unit.type_id, unit.id[0:4], unit.id))
def set_storage_path(self, filename=None): """ Set the storage path. This is a total hack to support existing single-file units with a _storage_path that includes the file name. :param filename: An optional filename to appended to the path. :rtype filename: str """ path = FileStorage.get_path(self) if filename: if not os.path.isabs(filename): path = os.path.join(path, filename) else: raise ValueError(_('must be relative path')) self._storage_path = path
def pre_save_signal(cls, sender, document, **kwargs): """ The signal that is triggered before a unit is saved, this is used to support the legacy behavior of generating the unit id and setting the last_updated timestamp :param sender: sender class :type sender: object :param document: Document that sent the signal :type document: FileContentUnit """ super(FileContentUnit, cls).pre_save_signal(sender, document, **kwargs) if not document._source_location: # no content return with FileStorage() as storage: storage.put(document, document._source_location)
def test_get_path(self, config): storage_dir = '/tmp/storage' config.get = lambda s, p: { 'server': { 'storage_dir': storage_dir } }[s][p] unit = Mock(id='0123456789', type_id='ABC') # test path = FileStorage.get_path(unit) # validation self.assertEqual( path, os.path.join(storage_dir, 'content', 'units', unit.type_id, unit.id[0:4], unit.id))
def test_get_path(self, config, sha256): storage_dir = '/tmp/storage' digest = '0123456789' config.get = lambda s, p: {'server': {'storage_dir': storage_dir}}[s][p] unit = Mock(type_id='ABC') unit.unit_key_as_digest.return_value = digest # test path = FileStorage.get_path(unit) # validation unit.unit_key_as_digest.assert_called_once_with(sha256.return_value) self.assertEqual( path, os.path.join(storage_dir, 'content', 'units', unit.type_id, digest[0:2], digest[2:]))
def download_succeeded(self, report): """ Marks the individual file for the unit as downloaded and moves it into its final storage location if its checksum value matches the value in the catalog entry (if present). Inherited from DownloadEventListener. :param report: the report associated with the download request. :type report: nectar.report.DownloadReport """ # Reload the content unit unit_model = plugin_api.get_unit_model_by_id(report.data[TYPE_ID]) unit_qs = unit_model.objects.filter(id=report.data[UNIT_ID]) content_unit = unit_qs.only('_content_type_id', 'id', '_last_updated').get() path_entry = report.data[UNIT_FILES][report.destination] # Validate the file and update the progress. catalog_entry = path_entry[CATALOG_ENTRY] try: self.validate_file(report.destination, catalog_entry.checksum_algorithm, catalog_entry.checksum) relative_path = os.path.relpath(catalog_entry.path, FileStorage.get_path(content_unit)) if len(report.data[UNIT_FILES]) == 1: # If the unit is single-file, update the storage path to point to the file content_unit.set_storage_path(relative_path) unit_qs.update_one( set___storage_path=content_unit._storage_path) content_unit.import_content(report.destination) else: content_unit.import_content(report.destination, location=relative_path) self.progress_successes += 1 path_entry[PATH_DOWNLOADED] = True except (InvalidChecksumType, VerificationException, IOError), e: _logger.debug( _('Download of {path} failed: {reason}.').format( path=catalog_entry.path, reason=str(e))) path_entry[PATH_DOWNLOADED] = False self.progress_failures += 1
def download_succeeded(self, report): """ Marks the individual file for the unit as downloaded and moves it into its final storage location if its checksum value matches the value in the catalog entry (if present). Inherited from DownloadEventListener. :param report: the report associated with the download request. :type report: nectar.report.DownloadReport """ # Reload the content unit unit_model = plugin_api.get_unit_model_by_id(report.data[TYPE_ID]) unit_qs = unit_model.objects.filter(id=report.data[UNIT_ID]) content_unit = unit_qs.only('_content_type_id', 'id', '_last_updated').get() path_entry = report.data[UNIT_FILES][report.destination] # Validate the file and update the progress. catalog_entry = path_entry[CATALOG_ENTRY] try: self.validate_file( report.destination, catalog_entry.checksum_algorithm, catalog_entry.checksum ) relative_path = os.path.relpath( catalog_entry.path, FileStorage.get_path(content_unit) ) if len(report.data[UNIT_FILES]) == 1: # If the unit is single-file, update the storage path to point to the file content_unit.set_storage_path(relative_path) unit_qs.update_one(set___storage_path=content_unit._storage_path) content_unit.import_content(report.destination) else: content_unit.import_content(report.destination, location=relative_path) self.progress_successes += 1 path_entry[PATH_DOWNLOADED] = True except (InvalidChecksumType, VerificationException, IOError), e: _logger.debug(_('Download of {path} failed: {reason}.').format( path=catalog_entry.path, reason=str(e))) path_entry[PATH_DOWNLOADED] = False self.progress_failures += 1
def test_get_path(self, config, sha256): storage_dir = '/tmp/storage' digest = '0123456789' config.get = lambda s, p: { 'server': { 'storage_dir': storage_dir } }[s][p] unit = Mock(type_id='ABC') unit.unit_key_as_digest.return_value = digest # test path = FileStorage.get_path(unit) # validation unit.unit_key_as_digest.assert_called_once_with(sha256.return_value) self.assertEqual( path, os.path.join(storage_dir, 'content', 'units', unit.type_id, digest[0:2], digest[2:]))
def test_put_file_incorrect_size(self, _mkdir, shutil, tempfile, close, remove, rename): path_in = '/tmp/test' temp_destination = '/some/file/path' unit = Mock(id='123', storage_path='/tmp/storage') storage = FileStorage() tempfile.mkstemp.return_value = ('fd', temp_destination) unit.verify_size.side_effect = verification.VerificationException(22) # test self.assertRaises(verification.VerificationException, storage.put, unit, path_in) # validation _mkdir.assert_called_once_with(os.path.dirname(unit.storage_path)) tempfile.mkstemp.assert_called_once_with( dir=os.path.dirname(unit.storage_path)) close.assert_called_once_with('fd') shutil.copy.assert_called_once_with(path_in, temp_destination) unit.verify_size.assert_called_once_with(temp_destination) remove.assert_called_once_with(temp_destination) self.assertFalse(rename.called)
def test_get(self): storage = FileStorage() storage.get(None) # just for coverage