def __init__(self, working_dir, conduit, config, repo):
"""
:param working_dir: working directory to create the filelists.xml.gz in
:type working_dir: str
:param conduit: The conduit to get api calls
:type conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
:param config: Pulp configuration for the distributor
:type config: pulp.plugins.config.PluginCallConfiguration
:param repo: Pulp managed repository
:type repo: pulp.plugins.model.Repository
"""
self.repo_id = repo.id
metadata_file_path = os.path.join(working_dir,
configuration.get_redirect_file_name(repo))
super(RedirectFileContext, self).__init__(metadata_file_path)
scratchpad = conduit.get_repo_scratchpad()
tag_list = scratchpad.get(u'tags', [])
self.tags = self.convert_tag_list_to_dict(tag_list)
self.registry = configuration.get_repo_registry_id(repo, config)
self.redirect_url = configuration.get_redirect_url(config, repo, 'v1')
if config.get('protected', False):
self.protected = "true"
else:
self.protected = "false"
def process_main(self):
"""
Publish the JSON file for Crane.
"""
registry = configuration.get_repo_registry_id(self.get_repo(),
self.get_config())
redirect_url = configuration.get_redirect_url(self.get_config(),
self.get_repo(), 'v2')
schema2_data = self.redirect_data[2]
manifest_list_data = self.redirect_data['list']
manifest_list_amd64 = self.redirect_data['amd64']
redirect_data = {
'type': 'pulp-docker-redirect',
'version': 4,
'repository': self.get_repo().id,
'repo-registry-id': registry,
'url': redirect_url,
'protected': self.get_config().get('protected', False),
'schema2_data': list(schema2_data),
'manifest_list_data': list(manifest_list_data),
'manifest_list_amd64_tags': manifest_list_amd64
}
misc.mkdir(os.path.dirname(self.app_publish_location))
with open(self.app_publish_location, 'w') as app_file:
app_file.write(json.dumps(redirect_data))
def finalize(self):
"""
Write the Tag list file so that clients can retrieve the list of available Tags.
"""
tags_path = os.path.join(self.parent.get_working_dir(), 'tags')
misc.mkdir(tags_path)
with open(os.path.join(tags_path, 'list'), 'w') as list_file:
tag_data = {
'name': configuration.get_repo_registry_id(self.get_repo(), self.get_config()),
'tags': list(self._tag_names)}
list_file.write(json.dumps(tag_data))
# We don't need the tag names anymore
del self._tag_names
def finalize(self):
"""
Write the Tag list file so that clients can retrieve the list of available Tags.
"""
tags_path = os.path.join(self.parent.get_working_dir(), 'tags')
misc.mkdir(tags_path)
with open(os.path.join(tags_path, 'list'), 'w') as list_file:
tag_data = {
'name': configuration.get_repo_registry_id(self.get_repo(), self.get_config()),
'tags': list(self._tag_names)}
list_file.write(json.dumps(tag_data))
# We don't need the tag names anymore
del self._tag_names
def process_main(self, item=False):
repo_registry_id = configuration.get_repo_registry_id(
self.get_repo(), self.get_config())
for img_name, tags in self.parent.imgname_to_tags.items():
dest_name = os.path.join(self.get_working_dir(), img_name, "tags",
constants.MANIFEST_LIST_TYPE)
repo_name = os.path.join(repo_registry_id, img_name)
tag_data = {
'name': repo_name,
'tags': sorted(tags),
}
misc.mkdir(os.path.dirname(dest_name))
with open(dest_name, 'w') as list_file:
list_file.write(json.dumps(tag_data))
def process_main(self):
"""
Publish the JSON file for Crane.
"""
registry = configuration.get_repo_registry_id(self.get_repo(), self.get_config())
redirect_url = configuration.get_redirect_url(self.get_config(), self.get_repo(), 'v2')
redirect_data = {
'type': 'pulp-docker-redirect', 'version': 2, 'repository': self.get_repo().id,
'repo-registry-id': registry, 'url': redirect_url,
'protected': self.get_config().get('protected', False)}
misc.mkdir(os.path.dirname(self.app_publish_location))
with open(self.app_publish_location, 'w') as app_file:
app_file.write(json.dumps(redirect_data))
def process_main(self):
"""
Publish the JSON file for Crane.
"""
repo = self.get_repo()
config = self.get_config()
registry_prefix = configuration.get_repo_registry_id(repo, config)
global_app_publish_dir = os.path.join(
configuration.get_app_publish_dir(config,
self.parent.docker_api_version),
registry_prefix)
self.parent.atomic_publish_step.publish_locations.append(
("app", global_app_publish_dir))
# Compute image references: tags and manifest IDs
img_refs = dict(
(img_name, set(tags))
for img_name, tags in self.parent.imgname_to_tags.items())
for img_name, refs in self.parent.imgname_to_mfid.items():
img_refs.setdefault(img_name, set()).update(refs)
redirect_url = configuration.get_redirect_url(
config, repo, self.parent.docker_api_version)
# Defaults
redirect_tmpl = {
'type': 'pulp-docker-redirect',
'version': 4,
'repository': self.get_repo().id,
'protected': config.get('protected', False),
'manifest_list_data': [],
'manifest_list_amd64_tags': {}
}
working_dir = self.get_working_dir()
for img_name, refs in img_refs.items():
app_publish_path = os.path.join("app", "%s.json" % img_name)
app_publish_location = os.path.join(working_dir, app_publish_path)
misc.mkdir(os.path.dirname(app_publish_location))
redirect_data = dict(redirect_tmpl)
redirect_data['repo-registry-id'] = os.path.join(
registry_prefix, img_name)
redirect_data['url'] = os.path.join(redirect_url, img_name)
redirect_data['schema2_data'] = sorted(refs)
with open(app_publish_location, 'w') as app_file:
app_file.write(json.dumps(redirect_data))
def process_main(self):
repo = self.get_repo()
config = self.get_config()
registry_prefix = configuration.get_repo_registry_id(repo, config)
dest_name = os.path.join(self.get_working_dir(), 'image_list.json')
name_to_mfdata = dict()
for manifest_id, name_tags in sorted(
self.parent.manifest_to_imgname_to_tags.items()):
for imgname, tags in sorted(name_tags.items()):
name_to_mfdata.setdefault("%s/%s" % (registry_prefix, imgname),
[]).append(
dict(manifest_id=manifest_id,
tags=sorted(x[0]
for x in tags)))
images = []
for name, mfdata in sorted(name_to_mfdata.items()):
images.append(dict(name=name, manifests=mfdata))
with open(dest_name, "w") as fobj:
json.dump(dict(images=images), fobj)
def process_main(self):
"""
Create the list file and add the manifest tag links.
:param unit: The unit to process
:type unit: pulp_docker.common.models.Tag
"""
tags_path = os.path.join(self.parent.get_working_dir(), 'tags')
misc.mkdir(tags_path)
with open(os.path.join(tags_path, 'list'), 'w') as list_file:
tag_data = {
'name': configuration.get_repo_registry_id(self.get_repo(), self.get_config()),
'tags': list(self.parent.tags)}
list_file.write(json.dumps(tag_data))
# Add the links to make Manifests accessible by tags as well
for tag, unit in self.parent.tags.items():
self.parent.publish_manifests_step._create_symlink(
unit.storage_path,
os.path.join(self.parent.publish_manifests_step.get_manifests_directory(), tag))
def process_main(self):
"""
Publish the JSON file for Crane.
"""
registry = configuration.get_repo_registry_id(self.get_repo(),
self.get_config())
redirect_url = configuration.get_redirect_url(self.get_config(),
self.get_repo(), 'v2')
redirect_data = {
'type': 'pulp-docker-redirect',
'version': 2,
'repository': self.get_repo().id,
'repo-registry-id': registry,
'url': redirect_url,
'protected': self.get_config().get('protected', False)
}
misc.mkdir(os.path.dirname(self.app_publish_location))
with open(self.app_publish_location, 'w') as app_file:
app_file.write(json.dumps(redirect_data))
def _add_necesary_steps(self, date_filter=None, config=None):
"""
This method adds all the steps that are needed to accomplish an RPM rsync publish. This
includes:
Unit Query Step - selects units associated with the repo based on the date_filter and
creates relative symlinks
Tag Generation Step - creates relative symlinks for for all tags and writes out list file
Rsync Step (content units) - rsyncs content units from /var/lib/pulp/content to remote
server
Rsync Step (symlinks) - rsyncs symlinks from working directory to remote server
:param date_filter: Q object with start and/or end dates, or None if start and end dates
are not provided
:type date_filter: mongoengine.Q or types.NoneType
:param config: distributor configuration
:type config: pulp.plugins.config.PluginCallConfiguration
:return: None
"""
postdistributor = self._get_postdistributor()
repo_registry_id = configuration.get_repo_registry_id(
self.repo, postdistributor.config)
remote_repo_path = configuration.get_remote_repo_relative_path(
self.repo, self.config)
unit_models = {
constants.IMAGE_TYPE_ID: models.Image,
constants.MANIFEST_TYPE_ID: models.Manifest,
constants.MANIFEST_LIST_TYPE_ID: models.ManifestList,
constants.BLOB_TYPE_ID: models.Blob
}
for unit_type in DockerRsyncPublisher.REPO_CONTENT_TYPES:
gen_step = RSyncFastForwardUnitPublishStep(
"Unit query step (things)", [unit_models[unit_type]],
repo=self.repo,
repo_content_unit_q=date_filter,
remote_repo_path=remote_repo_path)
self.add_child(gen_step)
self.add_child(
PublishTagsForRsyncStep("Generate tags step",
repo=self.repo,
repo_content_unit_q=date_filter,
remote_repo_path=remote_repo_path,
repo_registry_id=repo_registry_id))
origin_dest_prefix = self.get_units_directory_dest_path()
origin_src_prefix = self.get_units_src_path()
self.add_child(
RSyncPublishStep("Rsync step (content units)",
self.content_unit_file_list,
origin_src_prefix,
origin_dest_prefix,
config=config))
# Stop here if distributor is only supposed to publish actual content
if self.get_config().flatten().get("content_units_only"):
return
self.add_child(
RSyncPublishStep("Rsync step (symlinks)",
self.symlink_list,
self.symlink_src,
remote_repo_path,
config=config,
links=True,
delete=self.config.get("delete")))
if constants.TAG_TYPE_ID in self.repo.content_unit_counts:
self.add_child(
RSyncPublishStep("Rsync step (tags list)", ["tags/list"],
os.path.join(self.get_working_dir(),
'.relative'),
remote_repo_path,
config=config,
links=True))
def _add_necesary_steps(self, date_filter=None, config=None):
"""
This method adds all the steps that are needed to accomplish an RPM rsync publish. This
includes:
Unit Query Step - selects units associated with the repo based on the date_filter and
creates relative symlinks
Tag Generation Step - creates relative symlinks for for all tags and writes out list file
Rsync Step (content units) - rsyncs content units from /var/lib/pulp/content to remote
server
Rsync Step (symlinks) - rsyncs symlinks from working directory to remote server
:param date_filter: Q object with start and/or end dates, or None if start and end dates
are not provided
:type date_filter: mongoengine.Q or types.NoneType
:param config: distributor configuration
:type config: pulp.plugins.config.PluginCallConfiguration
:return: None
"""
postdistributor = self._get_postdistributor()
repo_registry_id = configuration.get_repo_registry_id(self.repo, postdistributor.config)
remote_repo_path = configuration.get_remote_repo_relative_path(self.repo, self.config)
unit_info = {constants.IMAGE_TYPE_ID: {'extra_path': [''], 'model': models.Image},
constants.MANIFEST_TYPE_ID: {'extra_path': ['manifests'],
'model': models.Manifest},
constants.BLOB_TYPE_ID: {'extra_path': ['blobs'], 'model': models.Blob}}
for unit_type in DockerRsyncPublisher.REPO_CONTENT_TYPES:
unit_path = unit_info[unit_type]['extra_path']
gen_step = RSyncFastForwardUnitPublishStep("Unit query step (things)",
[unit_info[unit_type]['model']],
repo=self.repo,
repo_content_unit_q=date_filter,
remote_repo_path=remote_repo_path,
published_unit_path=unit_path)
self.add_child(gen_step)
self.add_child(PublishTagsForRsyncStep("Generate tags step",
repo=self.repo,
repo_content_unit_q=date_filter,
remote_repo_path=remote_repo_path,
repo_registry_id=repo_registry_id))
origin_dest_prefix = self.get_units_directory_dest_path()
origin_src_prefix = self.get_units_src_path()
self.add_child(RSyncPublishStep("Rsync step (content units)", self.content_unit_file_list,
origin_src_prefix, origin_dest_prefix,
config=config))
# Stop here if distributor is only supposed to publish actual content
if self.get_config().flatten().get("content_units_only"):
return
self.add_child(RSyncPublishStep("Rsync step (symlinks)",
self.symlink_list, self.symlink_src, remote_repo_path,
config=config, links=True,
delete=self.config.get("delete")))
if constants.TAG_TYPE_ID in self.repo.content_unit_counts:
self.add_child(RSyncPublishStep("Rsync step (tags list)", ["tags/list"],
os.path.join(self.get_working_dir(), '.relative'),
remote_repo_path, config=config, links=True))