def __init__(self, repo=None, conduit=None, config=None, working_dir=None):
"""
:param repo: repository to sync
:type repo: pulp.plugins.model.Repository
:param conduit: sync conduit to use
:type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param config: config object for the sync
:type config: pulp.plugins.config.PluginCallConfiguration
:param working_dir: full path to the directory in which transient files
should be stored before being moved into long-term
storage. This should be deleted by the caller after
step processing is complete.
:type working_dir: basestring
"""
super(SyncStep,
self).__init__(constants.SYNC_STEP_MAIN, repo, conduit, config,
working_dir, constants.IMPORTER_TYPE_ID)
self.description = _('Syncing Docker Repository')
# Unit keys, populated by GetMetadataStep
self.available_units = []
# populated by GetMetadataStep
self.tags = {}
self.validate(config)
download_config = nectar_config.importer_config_to_nectar_config(
config.flatten())
upstream_name = config.get(constants.CONFIG_KEY_UPSTREAM_NAME)
url = config.get(importer_constants.KEY_FEED)
# create a Repository object to interact with
self.index_repository = Repository(upstream_name, download_config, url,
working_dir)
self.add_child(GetMetadataStep(working_dir=working_dir))
# save this step so its "units_to_download" attribute can be accessed later
self.step_get_local_units = GetLocalImagesStep(
constants.IMPORTER_TYPE_ID, constants.IMAGE_TYPE_ID, ['image_id'],
working_dir)
self.add_child(self.step_get_local_units)
self.add_child(
DownloadStep(constants.SYNC_STEP_DOWNLOAD,
downloads=self.generate_download_requests(),
repo=repo,
config=config,
working_dir=working_dir,
description=_('Downloading remote files')))
self.add_child(SaveUnits(working_dir))
def __init__(self, repo=None, conduit=None, config=None,
working_dir=None):
"""
:param repo: repository to sync
:type repo: pulp.plugins.model.Repository
:param conduit: sync conduit to use
:type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param config: config object for the sync
:type config: pulp.plugins.config.PluginCallConfiguration
:param working_dir: full path to the directory in which transient files
should be stored before being moved into long-term
storage. This should be deleted by the caller after
step processing is complete.
:type working_dir: basestring
"""
super(SyncStep, self).__init__(constants.SYNC_STEP_MAIN, repo, conduit, config,
working_dir, constants.IMPORTER_TYPE_ID)
self.description = _('Syncing Docker Repository')
# Unit keys, populated by GetMetadataStep
self.available_units = []
# populated by GetMetadataStep
self.tags = {}
self.validate(config)
download_config = nectar_config.importer_config_to_nectar_config(config.flatten())
upstream_name = config.get(constants.CONFIG_KEY_UPSTREAM_NAME)
url = config.get(importer_constants.KEY_FEED)
# create a Repository object to interact with
self.index_repository = Repository(upstream_name, download_config, url, working_dir)
self.add_child(GetMetadataStep(working_dir=working_dir))
# save this step so its "units_to_download" attribute can be accessed later
self.step_get_local_units = GetLocalImagesStep(constants.IMPORTER_TYPE_ID,
constants.IMAGE_TYPE_ID,
['image_id'], working_dir)
self.add_child(self.step_get_local_units)
self.add_child(DownloadStep(constants.SYNC_STEP_DOWNLOAD,
downloads=self.generate_download_requests(),
repo=repo, config=config, working_dir=working_dir,
description=_('Downloading remote files')))
self.add_child(SaveUnits(working_dir))
def __init__(self, repo=None, conduit=None, config=None, **kwargs):
"""
:param repo: repository to sync
:type repo: pulp.plugins.model.Repository
:param conduit: sync conduit to use
:type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param config: config object for the sync
:type config: pulp.plugins.config.PluginCallConfiguration
"""
super(SyncStep, self).__init__(constants.SYNC_STEP_MAIN,
repo=repo, conduit=conduit,
config=config, plugin_type=constants.IMPORTER_TYPE_ID,
**kwargs)
self.description = _('Syncing Docker Repository')
# Unit keys, populated by GetMetadataStep
self.available_units = []
# populated by GetMetadataStep
self.tags = {}
self.validate(config)
download_config = nectar_config.importer_config_to_nectar_config(config.flatten())
upstream_name = config.get(constants.CONFIG_KEY_UPSTREAM_NAME)
url = config.get(importer_constants.KEY_FEED)
# create a Repository object to interact with
self.index_repository = Repository(upstream_name, download_config, url,
self.get_working_dir())
self.add_child(GetMetadataStep())
# save this step so its "units_to_download" attribute can be accessed later
self.step_get_local_units = GetLocalUnitsStep(constants.IMPORTER_TYPE_ID)
self.add_child(self.step_get_local_units)
self.add_child(DownloadStep(constants.SYNC_STEP_DOWNLOAD,
downloads=self.generate_download_requests(),
repo=repo, config=config,
description=_('Downloading remote files')))
self.add_child(SaveDockerUnits())
class SyncStep(PluginStep):
required_settings = (
constants.CONFIG_KEY_UPSTREAM_NAME,
importer_constants.KEY_FEED,
)
def __init__(self, repo=None, conduit=None, config=None, working_dir=None):
"""
:param repo: repository to sync
:type repo: pulp.plugins.model.Repository
:param conduit: sync conduit to use
:type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param config: config object for the sync
:type config: pulp.plugins.config.PluginCallConfiguration
:param working_dir: full path to the directory in which transient files
should be stored before being moved into long-term
storage. This should be deleted by the caller after
step processing is complete.
:type working_dir: basestring
"""
super(SyncStep,
self).__init__(constants.SYNC_STEP_MAIN, repo, conduit, config,
working_dir, constants.IMPORTER_TYPE_ID)
self.description = _('Syncing Docker Repository')
# Unit keys, populated by GetMetadataStep
self.available_units = []
# populated by GetMetadataStep
self.tags = {}
self.validate(config)
download_config = nectar_config.importer_config_to_nectar_config(
config.flatten())
upstream_name = config.get(constants.CONFIG_KEY_UPSTREAM_NAME)
url = config.get(importer_constants.KEY_FEED)
# create a Repository object to interact with
self.index_repository = Repository(upstream_name, download_config, url,
working_dir)
self.add_child(GetMetadataStep(working_dir=working_dir))
# save this step so its "units_to_download" attribute can be accessed later
self.step_get_local_units = GetLocalImagesStep(
constants.IMPORTER_TYPE_ID, constants.IMAGE_TYPE_ID, ['image_id'],
working_dir)
self.add_child(self.step_get_local_units)
self.add_child(
DownloadStep(constants.SYNC_STEP_DOWNLOAD,
downloads=self.generate_download_requests(),
repo=repo,
config=config,
working_dir=working_dir,
description=_('Downloading remote files')))
self.add_child(SaveUnits(working_dir))
@classmethod
def validate(cls, config):
"""
Ensure that any required settings have non-empty values.
:param config: config object for the sync
:type config: pulp.plugins.config.PluginCallConfiguration
:raises MissingValue: if any required sync setting is missing
"""
missing = []
for key in cls.required_settings:
if not config.get(key):
missing.append(key)
if missing:
raise MissingValue(missing)
def generate_download_requests(self):
"""
a generator that yields DownloadRequest objects based on which units
were determined to be needed. This looks at the GetLocalUnits step's
output, which includes a list of units that need their files downloaded.
:return: generator of DownloadRequest instances
:rtype: types.GeneratorType
"""
for unit_key in self.step_get_local_units.units_to_download:
image_id = unit_key['image_id']
destination_dir = os.path.join(self.get_working_dir(), image_id)
try:
os.makedirs(destination_dir, mode=0755)
except OSError, e:
# it's ok if the directory exists
if e.errno != errno.EEXIST:
raise
# we already retrieved the ancestry files for the tagged images, so
# some of these will already exist
if not os.path.exists(os.path.join(destination_dir, 'ancestry')):
yield self.index_repository.create_download_request(
image_id, 'ancestry', destination_dir)
yield self.index_repository.create_download_request(
image_id, 'json', destination_dir)
yield self.index_repository.create_download_request(
image_id, 'layer', destination_dir)
class SyncStep(PluginStep):
required_settings = (
constants.CONFIG_KEY_UPSTREAM_NAME,
importer_constants.KEY_FEED,
)
def __init__(self, repo=None, conduit=None, config=None,
working_dir=None):
"""
:param repo: repository to sync
:type repo: pulp.plugins.model.Repository
:param conduit: sync conduit to use
:type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param config: config object for the sync
:type config: pulp.plugins.config.PluginCallConfiguration
:param working_dir: full path to the directory in which transient files
should be stored before being moved into long-term
storage. This should be deleted by the caller after
step processing is complete.
:type working_dir: basestring
"""
super(SyncStep, self).__init__(constants.SYNC_STEP_MAIN, repo, conduit, config,
working_dir, constants.IMPORTER_TYPE_ID)
self.description = _('Syncing Docker Repository')
# Unit keys, populated by GetMetadataStep
self.available_units = []
# populated by GetMetadataStep
self.tags = {}
self.validate(config)
download_config = nectar_config.importer_config_to_nectar_config(config.flatten())
upstream_name = config.get(constants.CONFIG_KEY_UPSTREAM_NAME)
url = config.get(importer_constants.KEY_FEED)
# create a Repository object to interact with
self.index_repository = Repository(upstream_name, download_config, url, working_dir)
self.add_child(GetMetadataStep(working_dir=working_dir))
# save this step so its "units_to_download" attribute can be accessed later
self.step_get_local_units = GetLocalImagesStep(constants.IMPORTER_TYPE_ID,
constants.IMAGE_TYPE_ID,
['image_id'], working_dir)
self.add_child(self.step_get_local_units)
self.add_child(DownloadStep(constants.SYNC_STEP_DOWNLOAD,
downloads=self.generate_download_requests(),
repo=repo, config=config, working_dir=working_dir,
description=_('Downloading remote files')))
self.add_child(SaveUnits(working_dir))
@classmethod
def validate(cls, config):
"""
Ensure that any required settings have non-empty values.
:param config: config object for the sync
:type config: pulp.plugins.config.PluginCallConfiguration
:raises MissingValue: if any required sync setting is missing
"""
missing = []
for key in cls.required_settings:
if not config.get(key):
missing.append(key)
if missing:
raise MissingValue(missing)
def generate_download_requests(self):
"""
a generator that yields DownloadRequest objects based on which units
were determined to be needed. This looks at the GetLocalUnits step's
output, which includes a list of units that need their files downloaded.
:return: generator of DownloadRequest instances
:rtype: types.GeneratorType
"""
for unit_key in self.step_get_local_units.units_to_download:
image_id = unit_key['image_id']
destination_dir = os.path.join(self.get_working_dir(), image_id)
try:
os.makedirs(destination_dir, mode=0755)
except OSError, e:
# it's ok if the directory exists
if e.errno != errno.EEXIST:
raise
# we already retrieved the ancestry files for the tagged images, so
# some of these will already exist
if not os.path.exists(os.path.join(destination_dir, 'ancestry')):
yield self.index_repository.create_download_request(image_id, 'ancestry',
destination_dir)
yield self.index_repository.create_download_request(image_id, 'json', destination_dir)
yield self.index_repository.create_download_request(image_id, 'layer', destination_dir)
