def test_extract_metadata_no_metadata(self, mock_tarfile):
# Setup
self.module.name = 'no-metadata'
filename = os.path.join(self.module_dir, self.module.filename())
# Test
try:
metadata.extract_metadata(filename, self.tmp_dir)
self.fail()
except metadata.MissingModuleFile, e:
self.assertEqual(e.module_filename, filename)
def test_extract_metadata_no_metadata(self):
# Setup
self.module.name = 'no-metadata'
filename = os.path.join(self.module_dir, self.module.filename())
# Test
try:
metadata.extract_metadata(self.module, filename, self.tmp_dir)
self.fail()
except metadata.MissingModuleFile, e:
self.assertEqual(e.module_filename, filename)
def test_extract_metadata_no_metadata(self, mock_tarfile):
# Setup
self.module.name = 'no-metadata'
filename = os.path.join(self.module_dir, self.module.filename())
# Test
try:
metadata.extract_metadata(filename, self.tmp_dir)
self.fail()
except metadata.MissingMetadataFile, e:
self.assertEqual(e.error_code, error_codes.PUP0001)
self.assertTrue(isinstance(e, PulpCodedException))
def test_extract_metadata_bad_tarball(self):
# Setup
self.module.name = 'empty'
filename = os.path.join(self.module_dir, self.module.filename())
# Test
try:
metadata.extract_metadata(filename, self.tmp_dir)
self.fail()
except metadata.InvalidTarball, e:
self.assertEqual(e.error_code, error_codes.PUP0002)
self.assertTrue(isinstance(e, PulpCodedException))
def test_extract_metadata_bad_tarball(self):
# Setup
self.module.name = 'empty'
filename = os.path.join(self.module_dir, self.module.filename())
# Test
try:
metadata.extract_metadata(filename, self.tmp_dir)
self.fail()
except metadata.InvalidTarball, e:
self.assertEqual(e.error_code, error_codes.PUP0002)
self.assertTrue(isinstance(e, PulpCodedException))
def test_extract_metadata_no_metadata(self, mock_tarfile):
# Setup
self.module.name = 'no-metadata'
filename = os.path.join(self.module_dir, self.module.filename())
# Test
try:
metadata.extract_metadata(filename, self.tmp_dir)
self.fail()
except metadata.MissingMetadataFile, e:
self.assertEqual(e.error_code, error_codes.PUP0001)
self.assertTrue(isinstance(e, PulpCodedException))
def test_extract_metadata_bad_tarball(self):
# Setup
self.module.name = 'empty'
filename = os.path.join(self.module_dir, self.module.filename())
# Test
try:
metadata.extract_metadata(filename, self.tmp_dir)
self.fail()
except metadata.ExtractionException, e:
self.assertEqual(e.module_filename, filename)
self.assertEqual(e.property_names[0], filename)
self.assertTrue(isinstance(e, InvalidValue))
def test_extract_metadata_bad_tarball(self):
# Setup
self.module.name = 'empty'
filename = os.path.join(self.module_dir, self.module.filename())
# Test
try:
metadata.extract_metadata(self.module, filename, self.tmp_dir)
self.fail()
except metadata.ExtractionException, e:
self.assertEqual(e.module_filename, filename)
self.assertEqual(e.property_names[0], filename)
self.assertTrue(isinstance(e, InvalidValue))
def test_extract_metadata(self):
# Setup
filename = os.path.join(self.module_dir, self.module.filename())
# Test
metadata.extract_metadata(self.module, filename, self.tmp_dir)
# Verify
self.assertEqual(self.module.name, 'valid')
self.assertEqual(self.module.version, '1.0.0')
self.assertEqual(self.module.author, 'jdob')
self._assert_test_module_metadata()
def test_extract_metadata(self):
# Setup
filename = os.path.join(self.module_dir, self.module.filename())
# Test
metadata.extract_metadata(self.module, filename, self.tmp_dir)
# Verify
self.assertEqual(self.module.name, 'valid')
self.assertEqual(self.module.version, '1.0.0')
self.assertEqual(self.module.author, 'jdob')
self._assert_test_module_metadata()
def handle_uploaded_unit(repo, type_id, unit_key, metadata, file_path, conduit):
"""
Handles an upload unit request to the importer. This call is responsible
for moving the unit from its temporary location where Pulp stored the
upload to the final storage location (as dictated by Pulp) for the unit.
This call will also update the database in Pulp to reflect the unit
and its association to the repository.
:param repo: repository into which the unit is being uploaded
:type repo: pulp.plugins.model.Repository
:param type_id: type of unit being uploaded
:type type_id: str
:param unit_key: unique identifier for the unit
:type unit_key: dict
:param metadata: extra data about the unit
:type metadata: dict
:param file_path: temporary location of the uploaded file
:type file_path: str
:param conduit: for calls back into Pulp
:type conduit: pulp.plugins.conduit.upload.UploadConduit
"""
if type_id != constants.TYPE_PUPPET_MODULE:
raise NotImplementedError()
# Create a module out of the uploaded metadata
combined = copy.copy(unit_key)
combined.update(metadata)
module = Module.from_dict(combined)
# Extract the metadata from the module
metadata_parser.extract_metadata(module, file_path, repo.working_dir)
# Create the Pulp unit
type_id = constants.TYPE_PUPPET_MODULE
unit_key = module.unit_key()
unit_metadata = module.unit_metadata()
relative_path = constants.STORAGE_MODULE_RELATIVE_PATH % module.filename()
unit = conduit.init_unit(type_id, unit_key, unit_metadata, relative_path)
# Copy from the upload temporary location into where Pulp wants it to live
shutil.copy(file_path, unit.storage_path)
# Save the unit into the destination repository
conduit.save_unit(unit)
def _add_new_module(self, downloader, module):
"""
Performs the tasks for downloading and saving a new unit in Pulp.
This method entirely skips modules that are already in the repository.
:param downloader: downloader instance to use for retrieving the unit
:type downloader: child of pulp_puppet.plugins.importers.downloaders.base.BaseDownloader
:param module: module to download and add
:type module: pulp_puppet.plugins.db.models.Module
"""
try:
# Download the bits
downloaded_filename = downloader.retrieve_module(self.progress_report, module)
# Extract the extra metadata into the module
metadata = metadata_module.extract_metadata(downloaded_filename,
self.repo.working_dir)
# Overwrite the author and name
metadata.update(Module.split_filename(metadata['name']))
# Create and save the Module
module = Module.from_metadata(metadata)
module.set_storage_path(os.path.basename(downloaded_filename))
try:
module.save_and_import_content(downloaded_filename)
except NotUniqueError:
module = module.__class__.objects.get(**module.unit_key)
# Associate the module with the repo
repo_controller.associate_single_unit(self.repo.repo_obj, module)
finally:
downloader.cleanup_module(module)
def test_retrieve_module(self):
module = model.Module('php', '0.2.1', 'thias')
# Test
self.downloader.retrieve_module(self.mock_progress_report, module)
# Verify
module_dir = os.path.join(self.working_dir, DOWNLOAD_TMP_DIR)
expected_file = os.path.join(module_dir, module.filename())
self.assertTrue(os.path.exists(expected_file))
# Extract the metadata to make sure the tar is valid and we can open it
metadata.extract_metadata(module, expected_file, self.working_dir)
# Spot check that something from the metadata was stuffed into the module
self.assertTrue(module.checksums is not None)
def test_retrieve_module(self):
module = model.Module('php', '0.2.1', 'thias')
# Test
self.downloader.retrieve_module(self.mock_progress_report, module)
# Verify
module_dir = os.path.join(self.working_dir, DOWNLOAD_TMP_DIR)
expected_file = os.path.join(module_dir, module.filename())
self.assertTrue(os.path.exists(expected_file))
# Extract the metadata to make sure the tar is valid and we can open it
metadata.extract_metadata(module, expected_file, self.working_dir)
# Spot check that something from the metadata was stuffed into the module
self.assertTrue(module.checksums is not None)
def _add_new_module(self, downloader, module):
"""
Performs the tasks for downloading and saving a new unit in Pulp.
This method entirely skips modules that are already in the repository.
:param downloader: downloader instance to use for retrieving the unit
:type downloader: child of pulp_puppet.plugins.importers.downloaders.base.BaseDownloader
:param module: module to download and add
:type module: pulp_puppet.plugins.db.models.Module
"""
try:
# Download the bits
downloaded_filename = downloader.retrieve_module(
self.progress_report, module)
# Extract the extra metadata into the module
metadata = metadata_module.extract_metadata(
downloaded_filename, self.repo.working_dir)
# Overwrite the author and name
metadata.update(Module.split_filename(metadata['name']))
# Create and save the Module
module = Module.from_metadata(metadata)
module.set_storage_path(os.path.basename(downloaded_filename))
try:
module.save_and_import_content(downloaded_filename)
except NotUniqueError:
module = module.__class__.objects.get(**module.unit_key)
# Associate the module with the repo
repo_controller.associate_single_unit(self.repo.repo_obj, module)
finally:
downloader.cleanup_module(module)
def test_extract_metadata_non_standard_packaging(self):
# Setup
self.module = Module('misnamed', '1.0.0', 'ldob')
self.module_dir = os.path.join(DATA_DIR, 'bad-modules')
filename = os.path.join(self.module_dir, self.module.filename())
# Test
metadata.extract_metadata(self.module, filename, self.tmp_dir)
# Verify - contains the same module as jdob-valid-1.0.0, so this is safe
self.assertEqual(self.module.name, 'misnamed')
self.assertEqual(self.module.version, '1.0.0')
self.assertEqual(self.module.author, 'ldob')
self._assert_test_module_metadata()
extraction_root = os.path.join(self.tmp_dir, self.module.author)
self.assertTrue(not os.path.exists(extraction_root))
def test_extract_metadata_non_standard_packaging(self):
# Setup
self.module = Module('misnamed', '1.0.0', 'ldob')
self.module_dir = os.path.join(DATA_DIR, 'bad-modules')
filename = os.path.join(self.module_dir, self.module.filename())
# Test
metadata.extract_metadata(self.module, filename, self.tmp_dir)
# Verify - contains the same module as jdob-valid-1.0.0, so this is safe
self.assertEqual(self.module.name, 'misnamed')
self.assertEqual(self.module.version, '1.0.0')
self.assertEqual(self.module.author, 'ldob')
self._assert_test_module_metadata()
extraction_root = os.path.join(self.tmp_dir, self.module.author)
self.assertTrue(not os.path.exists(extraction_root))
def _add_new_module(self, downloader, module):
"""
Performs the tasks for downloading and saving a new unit in Pulp.
:param downloader: downloader instance to use for retrieving the unit
:param module: module instance to download
:type module: Module
"""
# Initialize the unit in Pulp
type_id = constants.TYPE_PUPPET_MODULE
unit_key = module.unit_key()
unit_metadata = {} # populated later but needed for the init call
relative_path = constants.STORAGE_MODULE_RELATIVE_PATH % module.filename(
)
unit = self.sync_conduit.init_unit(type_id, unit_key, unit_metadata,
relative_path)
try:
if not self._module_exists(unit.storage_path):
# Download the bits
downloaded_filename = downloader.retrieve_module(
self.progress_report, module)
# Copy them to the final location
shutil.copy(downloaded_filename, unit.storage_path)
# Extract the extra metadata into the module
metadata.extract_metadata(module, unit.storage_path,
self.repo.working_dir)
# Update the unit with the extracted metadata
unit.metadata = module.unit_metadata()
# Save the unit and associate it to the repository
self.sync_conduit.save_unit(unit)
finally:
# Clean up the temporary module
downloader.cleanup_module(module)
def _add_new_module(self, downloader, module):
"""
Performs the tasks for downloading and saving a new unit in Pulp.
:param downloader: downloader instance to use for retrieving the unit
:param module: module instance to download
:type module: Module
"""
# Initialize the unit in Pulp
type_id = constants.TYPE_PUPPET_MODULE
unit_key = module.unit_key()
unit_metadata = {} # populated later but needed for the init call
relative_path = constants.STORAGE_MODULE_RELATIVE_PATH % module.filename()
unit = self.sync_conduit.init_unit(type_id, unit_key, unit_metadata,
relative_path)
try:
if not self._module_exists(unit.storage_path):
# Download the bits
downloaded_filename = downloader.retrieve_module(self.progress_report, module)
# Copy them to the final location
shutil.copy(downloaded_filename, unit.storage_path)
# Extract the extra metadata into the module
metadata.extract_metadata(module, unit.storage_path, self.repo.working_dir)
# Update the unit with the extracted metadata
unit.metadata = module.unit_metadata()
# Save the unit and associate it to the repository
self.sync_conduit.save_unit(unit)
finally:
# Clean up the temporary module
downloader.cleanup_module(module)
def handle_uploaded_unit(repo, type_id, unit_key, metadata, file_path,
conduit):
"""
Handles an upload unit request to the importer. This call is responsible
for moving the unit from its temporary location where Pulp stored the
upload to the final storage location (as dictated by Pulp) for the unit.
This call will also update the database in Pulp to reflect the unit
and its association to the repository.
:param repo: repository into which the unit is being uploaded
:type repo: pulp.plugins.model.Repository
:param type_id: type of unit being uploaded
:type type_id: str
:param unit_key: unique identifier for the unit
:type unit_key: dict
:param metadata: extra data about the unit
:type metadata: dict
:param file_path: temporary location of the uploaded file
:type file_path: str
:param conduit: for calls back into Pulp
:type conduit: pulp.plugins.conduit.upload.UploadConduit
"""
if type_id != constants.TYPE_PUPPET_MODULE:
raise NotImplementedError()
# Extract the metadata from the module
extracted_data = metadata_parser.extract_metadata(file_path,
repo.working_dir)
# Overwrite the author and name
extracted_data.update(Module.split_filename(extracted_data['name']))
uploaded_module = Module.from_metadata(extracted_data)
# rename the file so it has the original module name
new_file_path = os.path.join(os.path.dirname(file_path),
uploaded_module.puppet_standard_filename())
shutil.move(file_path, new_file_path)
uploaded_module.set_storage_path(os.path.basename(new_file_path))
try:
uploaded_module.save_and_import_content(new_file_path)
except NotUniqueError:
uploaded_module = uploaded_module.__class__.objects.get(
**uploaded_module.unit_key)
repo_controller.associate_single_unit(repo.repo_obj, uploaded_module)
return {'success_flag': True, 'summary': '', 'details': {}}
def test_extract_metadata_no_module(self, mkdtemp):
# Setup
filename = os.path.join(self.module_dir, self.module.filename())
extraction_dir = os.path.join(self.tmp_dir, "1234")
mkdtemp.return_value = extraction_dir
metadata_json = metadata.extract_metadata(filename, self.tmp_dir)
self.module = Module.from_json(metadata_json)
# Verify
self.assertEqual(self.module.name, 'valid')
self.assertEqual(self.module.version, '1.0.0')
self.assertEqual(self.module.author, 'jdob')
self._assert_test_module_metadata()
self.assertTrue(not os.path.exists(extraction_dir))
def test_extract_metadata_no_module(self, mkdtemp):
# Setup
filename = os.path.join(self.module_dir, self.module.filename())
extraction_dir = os.path.join(self.tmp_dir, "1234")
mkdtemp.return_value = extraction_dir
metadata_json = metadata.extract_metadata(filename, self.tmp_dir)
self.module = Module.from_json(metadata_json)
# Verify
self.assertEqual(self.module.name, 'valid')
self.assertEqual(self.module.version, '1.0.0')
self.assertEqual(self.module.author, 'jdob')
self._assert_test_module_metadata()
self.assertTrue(not os.path.exists(extraction_dir))
def handle_uploaded_unit(repo, type_id, unit_key, metadata, file_path, conduit):
"""
Handles an upload unit request to the importer. This call is responsible
for moving the unit from its temporary location where Pulp stored the
upload to the final storage location (as dictated by Pulp) for the unit.
This call will also update the database in Pulp to reflect the unit
and its association to the repository.
:param repo: repository into which the unit is being uploaded
:type repo: pulp.plugins.model.Repository
:param type_id: type of unit being uploaded
:type type_id: str
:param unit_key: unique identifier for the unit
:type unit_key: dict
:param metadata: extra data about the unit
:type metadata: dict
:param file_path: temporary location of the uploaded file
:type file_path: str
:param conduit: for calls back into Pulp
:type conduit: pulp.plugins.conduit.upload.UploadConduit
"""
if type_id != constants.TYPE_PUPPET_MODULE:
raise NotImplementedError()
# Extract the metadata from the module
extracted_data = metadata_parser.extract_metadata(file_path, repo.working_dir)
# Overwrite the author and name
extracted_data.update(Module.split_filename(extracted_data['name']))
uploaded_module = Module.from_metadata(extracted_data)
# rename the file so it has the original module name
new_file_path = os.path.join(os.path.dirname(file_path),
uploaded_module.puppet_standard_filename())
shutil.move(file_path, new_file_path)
uploaded_module.set_storage_path(os.path.basename(new_file_path))
try:
uploaded_module.save_and_import_content(new_file_path)
except NotUniqueError:
uploaded_module = uploaded_module.__class__.objects.get(**uploaded_module.unit_key)
repo_controller.associate_single_unit(repo.repo_obj, uploaded_module)
return {'success_flag': True, 'summary': '', 'details': {}}
def test_extract_metadata_non_standard_packaging(self, mkdtemp):
# Setup
self.module = Module('misnamed', '1.0.0', 'ldob')
self.module_dir = os.path.join(DATA_DIR, 'bad-modules')
filename = os.path.join(self.module_dir, self.module.filename())
extraction_dir = os.path.join(self.tmp_dir, "test")
mkdtemp.return_value = extraction_dir
# Test
metadata_json = metadata.extract_metadata(filename, self.tmp_dir, self.module)
self.module.update_from_dict(metadata_json)
# Verify - contains the same module as jdob-valid-1.0.0, so this is safe
self.assertEqual(self.module.name, 'misnamed')
self.assertEqual(self.module.version, '1.0.0')
self.assertEqual(self.module.author, 'ldob')
self._assert_test_module_metadata()
self.assertTrue(not os.path.exists(extraction_dir))
def test_extract_metadata_non_standard_packaging(self, mkdtemp):
# Setup
self.module = Module('misnamed', '1.0.0', 'ldob')
self.module_dir = os.path.join(DATA_DIR, 'bad-modules')
filename = os.path.join(self.module_dir, self.module.filename())
extraction_dir = os.path.join(self.tmp_dir, "test")
mkdtemp.return_value = extraction_dir
# Test
metadata_json = metadata.extract_metadata(filename, self.tmp_dir)
self.module.update_from_dict(metadata_json)
# Verify - contains the same module as jdob-valid-1.0.0, so this is safe
self.assertEqual(self.module.name, 'misnamed')
self.assertEqual(self.module.version, '1.0.0')
self.assertEqual(self.module.author, 'ldob')
self._assert_test_module_metadata()
self.assertTrue(not os.path.exists(extraction_dir))