def import_content(self, path, location=None):
"""
The parent class promises to import a content file into platform storage.
The (optional) *location* may be used to specify a path within the unit
storage where the content is to be stored.
For example:
import_content('/tmp/file') will store 'file' at: _storage_path
import_content('/tmp/file', 'a/b/c) will store 'file' at: _storage_path/a/b/c
In addition to the parent behavior, this overridden method calculates the
checksum after moving the content to permanent storage if it has not already
been provided.
:param path: The absolute path to the file to be imported.
:type path: str
:param location: The (optional) location within the unit storage path
where the content is to be stored.
:type location: str
:raises PulpCodedException: PLP0036 if the unit has not been saved.
:raises PulpCodedException: PLP0037 if *path* is not an existing file.
"""
super(Module, self).import_content(path, location=location)
if self.checksum is None:
self.checksum = metadata_parser.calculate_checksum(self._storage_path)
self.save()
def handle_uploaded_unit(repo, type_id, unit_key, metadata, file_path,
conduit):
"""
Handles an upload unit request to the importer. This call is responsible
for moving the unit from its temporary location where Pulp stored the
upload to the final storage location (as dictated by Pulp) for the unit.
This call will also update the database in Pulp to reflect the unit
and its association to the repository.
:param repo: repository into which the unit is being uploaded
:type repo: pulp.plugins.model.Repository
:param type_id: type of unit being uploaded
:type type_id: str
:param unit_key: unique identifier for the unit
:type unit_key: dict
:param metadata: extra data about the unit
:type metadata: dict
:param file_path: temporary location of the uploaded file
:type file_path: str
:param conduit: for calls back into Pulp
:type conduit: pulp.plugins.conduit.upload.UploadConduit
"""
if type_id != constants.TYPE_PUPPET_MODULE:
raise NotImplementedError()
# Create a module with unit_key if supplied
initial_module = None
if unit_key:
initial_module = Module.from_dict(unit_key)
# Extract the metadata from the module
extracted_data = metadata_parser.extract_metadata(file_path,
repo.working_dir,
initial_module)
checksum = metadata_parser.calculate_checksum(file_path)
# Create a module from the metadata
module = Module.from_json(extracted_data)
module.checksum = checksum
# Create the Pulp unit
type_id = constants.TYPE_PUPPET_MODULE
unit_key = module.unit_key()
unit_metadata = module.unit_metadata()
relative_path = constants.STORAGE_MODULE_RELATIVE_PATH % module.filename()
unit = conduit.init_unit(type_id, unit_key, unit_metadata, relative_path)
# Copy from the upload temporary location into where Pulp wants it to live
shutil.copy(file_path, unit.storage_path)
# Save the unit into the destination repository
conduit.save_unit(unit)
return {'success_flag': True, 'summary': '', 'details': {}}
def handle_uploaded_unit(repo, type_id, unit_key, metadata, file_path, conduit):
"""
Handles an upload unit request to the importer. This call is responsible
for moving the unit from its temporary location where Pulp stored the
upload to the final storage location (as dictated by Pulp) for the unit.
This call will also update the database in Pulp to reflect the unit
and its association to the repository.
:param repo: repository into which the unit is being uploaded
:type repo: pulp.plugins.model.Repository
:param type_id: type of unit being uploaded
:type type_id: str
:param unit_key: unique identifier for the unit
:type unit_key: dict
:param metadata: extra data about the unit
:type metadata: dict
:param file_path: temporary location of the uploaded file
:type file_path: str
:param conduit: for calls back into Pulp
:type conduit: pulp.plugins.conduit.upload.UploadConduit
"""
if type_id != constants.TYPE_PUPPET_MODULE:
raise NotImplementedError()
# Create a module with unit_key if supplied
initial_module = None
if unit_key:
initial_module = Module.from_dict(unit_key)
# Extract the metadata from the module
extracted_data = metadata_parser.extract_metadata(file_path, repo.working_dir, initial_module)
checksum = metadata_parser.calculate_checksum(file_path)
# Create a module from the metadata
module = Module.from_json(extracted_data)
module.checksum = checksum
# Create the Pulp unit
type_id = constants.TYPE_PUPPET_MODULE
unit_key = module.unit_key()
unit_metadata = module.unit_metadata()
relative_path = constants.STORAGE_MODULE_RELATIVE_PATH % module.filename()
unit = conduit.init_unit(type_id, unit_key, unit_metadata, relative_path)
# Copy from the upload temporary location into where Pulp wants it to live
shutil.copy(file_path, unit.storage_path)
# Save the unit into the destination repository
conduit.save_unit(unit)
return {'success_flag': True, 'summary': '', 'details': {}}
def migrate(*args, **kwargs):
"""
for each puppet module, calculate a checksum for the source file on the filesystem
"""
query_manager = ContentQueryManager()
collection = query_manager.get_content_unit_collection(type_id=constants.TYPE_PUPPET_MODULE)
for puppet_unit in collection.find():
storage_path = puppet_unit['_storage_path']
checksum = metadata.calculate_checksum(storage_path)
puppet_unit['checksum'] = checksum
puppet_unit['checksum_type'] = constants.DEFAULT_HASHLIB
collection.save(puppet_unit)
_log.info("Migrated puppet modules to include checksum")
def migrate(*args, **kwargs):
"""
For each puppet module, calculate a checksum for the source file on the filesystem.
"""
query_manager = ContentQueryManager()
collection = query_manager.get_content_unit_collection(
type_id=constants.TYPE_PUPPET_MODULE)
for puppet_unit in collection.find():
storage_path = puppet_unit['_storage_path']
checksum = metadata.calculate_checksum(storage_path)
puppet_unit['checksum'] = checksum
puppet_unit['checksum_type'] = constants.DEFAULT_HASHLIB
collection.save(puppet_unit)
_log.info("Migrated puppet modules to include checksum")
def pre_save_signal(cls, sender, document, **kwargs):
"""
The signal that is triggered before a unit is saved. This is used to automatically
calculate the checksum field.
:param sender: sender class
:type sender: object
:param document: Document that sent the signal
:type document: FileContentUnit
"""
super(Module, cls).pre_save_signal(sender, document, **kwargs)
if document.checksum is None:
document.checksum = metadata_parser.calculate_checksum(document._storage_path)
# Checksums is expressed as a dict of files to checksum. This causes a problem in mongo
# since keys can't have periods in them, but file names clearly will. Translate this to a
# list of tuples to get around this.
if isinstance(document.checksums, dict):
document.checksums = [(k, v) for k, v in document.checksums.items()]
def test_checksum_calculation(self):
sample_module = os.path.join(self.module_dir, "jdob-valid-1.1.0.tar.gz")
sample_checksum = metadata.calculate_checksum(sample_module)
self.assertEquals(sample_checksum,
"108e8d1d9bb42c869344fc2d327c80e7f079d2ba0119da446a6a1c6659e0f0aa")
def test_checksum_calculation(self):
sample_module = os.path.join(self.module_dir, "jdob-valid-1.1.0.tar.gz")
sample_checksum = metadata.calculate_checksum(sample_module)
self.assertEquals(sample_checksum,
"108e8d1d9bb42c869344fc2d327c80e7f079d2ba0119da446a6a1c6659e0f0aa")