def run(self):
"""
Synchronize the repository with the remote repository.
This walks through the standard workflow that most sync operations want to follow. This
pattern is a recommended starting point for other plugins.
- Determine what is available remotely.
- Determine what is already in the local repository.
- Compare those two, and based on any importer settings or content-type-specific logic,
figure out what you want to add and remove from the local repository.
- Use a ChangeSet to make those changes happen.
"""
# Determine what is available remotely
self._fetch_manifest()
# Determine what is already in the repo
self._fetch_inventory()
# Based on the above two, figure out what we want to add and remove
self._find_delta()
additions = SizedIterable(self._build_additions(),
len(self._keys_to_add))
removals = SizedIterable(self._build_removals(),
len(self._keys_to_remove))
# Hand that to a ChangeSet, and we're done!
changeset = ChangeSet(self._importer,
additions=additions,
removals=removals)
changeset.apply_and_drain()
def build_removals(base_version, delta):
"""
Build the content to be removed.
Args:
base_version (RepositoryVersion): The base repository version.
delta (Delta): The set of Key to be added and removed.
Returns:
SizedIterable: The AnsibleRoleVersion to be removed from the repository.
"""
def generate():
for removals in BatchIterator(delta.removals):
q = Q()
for key in removals:
role = AnsibleRoleVersion.objects.get(name=key.name,
namespace=key.namespace)
q |= Q(ansibleroleversion__role_id=role.pk,
ansibleroleversion__version=key.version)
q_set = base_version.content.filter(q)
q_set = q_set.only('id')
for file in q_set:
yield file
return SizedIterable(generate(), len(delta.removals))
def build_additions(remote, manifest, delta):
"""
Build the content to be added.
Args:
remote (FileRemote): An remote.
manifest (Manifest): The downloaded manifest.
delta (Delta): The set of Key to be added and removed.
Returns:
SizedIterable: The PendingContent to be added to the repository.
"""
def generate():
for entry in manifest.read():
key = Key(relative_path=entry.relative_path, digest=entry.digest)
if key not in delta.additions:
continue
path = os.path.join(root_dir, entry.relative_path)
url = urlunparse(parsed_url._replace(path=path))
file = FileContent(relative_path=entry.relative_path,
digest=entry.digest)
artifact = Artifact(size=entry.size, sha256=entry.digest)
content = PendingContent(file,
artifacts={
PendingArtifact(
artifact, url,
entry.relative_path)
})
yield content
parsed_url = urlparse(remote.url)
root_dir = os.path.dirname(parsed_url.path)
return SizedIterable(generate(), len(delta.additions))
def _build_additions(delta, remote_metadata):
"""
Generate the content to be added.
Args:
delta (namedtuple): Tuple of content to add, and content to remove from the repository
remote_metadata (list): List of contentunit metadata
Returns:
The PythonPackageContent to be added to the repository.
"""
def generate():
for entry in remote_metadata:
if entry['filename'] not in delta.additions:
continue
url = entry.pop('url')
artifact = models.Artifact(sha256=entry.pop('sha256_digest'))
package = python_models.PythonPackageContent(**entry)
content = PendingContent(package,
artifacts={
PendingArtifact(
model=artifact,
url=url,
relative_path=entry['filename'])
})
yield content
return SizedIterable(generate(), len(delta.additions))
def build_additions(remote, roles, delta):
"""
Build the content to be added.
Args:
remote (AnsibleRemote): A remote.
roles (list): The list of role dict from Galaxy
delta (Delta): The set of Key to be added and removed.
Returns:
SizedIterable: The PendingContent to be added to the repository.
"""
def generate():
for metadata in roles:
role, _ = AnsibleRole.objects.get_or_create(
name=metadata['name'], namespace=metadata['namespace'])
for version in metadata['summary_fields']['versions']:
key = Key(name=metadata['name'],
namespace=metadata['namespace'],
version=version['name'])
if key not in delta.additions:
continue
url = GITHUB_URL % (metadata['github_user'],
metadata['github_repo'], version['name'])
role_version = AnsibleRoleVersion(version=version['name'],
role=role)
path = "%s/%s/%s.tar.gz" % (metadata['namespace'],
metadata['name'], version['name'])
artifact = Artifact()
content = PendingContent(
role_version,
artifacts={PendingArtifact(artifact, url, path)})
yield content
return SizedIterable(generate(), len(delta.additions))
def build_removals(base_version, delta):
"""
Build the content to be removed.
Args:
base_version (RepositoryVersion): The base repository version.
delta (Delta): The set of Key to be added and removed.
Returns:
SizedIterable: The FileContent to be removed from the repository.
"""
def generate():
for removals in BatchIterator(delta.removals):
q = Q()
for key in removals:
q |= Q(gemcontent__name=key.name,
gemcontent__version=key.version)
q_set = base_version.content.filter(q)
q_set = q_set.only('id')
for gem in q_set:
yield gem
return SizedIterable(generate(), len(delta.removals))
def build_removals(base_version, delta):
"""
Build the content to be removed.
Args:
base_version (RepositoryVersion): The base repository version.
delta (Delta): The set of Key to be added and removed.
Returns:
SizedIterable: The FileContent to be removed from the repository.
"""
def generate():
for removals in BatchIterator(delta.removals):
q = Q()
for key in removals:
q |= Q(filecontent__relative_path=key.relative_path,
filecontent__digest=key.digest)
q_set = base_version.content.filter(q)
q_set = q_set.only('id')
for file in q_set:
yield file
return SizedIterable(generate(), len(delta.removals))
def build_additions(remote, specs, delta):
"""
Build the content to be added.
Args:
remote (FileRemote): A remote.
specs (Specs): The downloaded specs.
delta (Delta): The set of Key to be added and removed.
Returns:
SizedIterable: The PendingContent to be added to the repository.
"""
def generate():
for key in delta.additions:
relative_path = os.path.join('gems',
key.name + '-' + key.version + '.gem')
path = os.path.join(root_dir, relative_path)
url = urlunparse(parsed_url._replace(path=path))
spec_relative_path = os.path.join(
'quick/Marshal.4.8',
key.name + '-' + key.version + '.gemspec.rz')
spec_path = os.path.join(root_dir, spec_relative_path)
spec_url = urlunparse(parsed_url._replace(path=spec_path))
gem = GemContent(name=key.name, version=key.version)
content = PendingContent(
gem,
artifacts={
PendingArtifact(Artifact(), url, relative_path),
PendingArtifact(Artifact(), spec_url, spec_relative_path),
})
yield content
parsed_url = urlparse(remote.url)
root_dir = os.path.dirname(parsed_url.path)
return SizedIterable(generate(), len(delta.additions))
def _build_removals(delta, version):
"""
Generate the content to be removed.
Args:
delta (namedtuple): Tuple of content to add, and content to remove from the repository
version (pulpcore.plugin.models.RepositoryVersion): Version of repository to remove
contents from
Returns:
The PythonPackageContent to be removed from the repository.
"""
def generate():
for removals in BatchIterator(delta.removals):
q = Q()
for content_key in removals:
q |= Q(pythonpackagecontent__filename=content_key)
q_set = version.content.filter(q)
q_set = q_set.only('id')
for content in q_set:
yield content
return SizedIterable(generate(), len(delta.removals))