def _read_desired_inventory(settings):
desired = {'modules': {}, 'hostgroups': {}, 'common': {}}
environments = get_names_of_declared_environments(settings)
for environmentname in environments:
try:
environment = read_environment_definition(
settings, environmentname)
# TODO: what if overrides empty? what if overrides,partition empty?
if 'overrides' in environment:
for partition in environment['overrides'].iterkeys():
if partition in ("modules", "hostgroups", "common"):
for name, override in \
environment['overrides'][partition].iteritems():
# prefixhash is equivalent to PREFIXhash, contrary to
# refs (branches, sic) which as case-sensitiive
if ref_is_commit(settings, override):
override = override.lower()
if name not in desired[partition]:
desired[partition][name] = [override]
else:
if override not in desired[partition][name]:
desired[partition][name].append(override)
except JensEnvironmentsError, error:
logging.error("Unable to process '%s' definition. Skipping" % \
environmentname)
continue # Just ignore, as won't be generated later on either.
def _expand_clones(settings, partition, name, inventory, inventory_lock,
new_refs, moved_refs, deleted_refs):
bare_path = _compose_bare_repository_path(settings, name, partition)
if new_refs:
logging.debug("Processing new refs of %s/%s (%s)..." % \
(partition, name, new_refs))
for refname in new_refs:
clone_path = _compose_clone_repository_path(settings, name, partition,
refname)
logging.info("Populating new ref '%s'" % clone_path)
try:
if ref_is_commit(settings, refname):
commit_id = refname.replace(settings.HASHPREFIX, '')
logging.debug("Will create a clone pointing to '%s'" %
commit_id)
git.clone(clone_path, "%s" % bare_path, shared=True)
git.reset(clone_path, commit_id, hard=True)
else:
git.clone(clone_path, "%s" % bare_path, branch=refname)
# Needs reset so the proxy notices about the change on the mutable
# http://docs.python.org/2.7/library/multiprocessing.html#managers
# Locking on the assignment is guarateed by the library, but
# additional locking is needed as A = A + 1 is a critical section.
if inventory_lock:
inventory_lock.acquire()
inventory[name] += [refname]
if inventory_lock:
inventory_lock.release()
except JensGitError, error:
if os.path.isdir(clone_path):
shutil.rmtree(clone_path)
logging.error("Unable to create clone '%s' (%s)" % \
(clone_path, error))
def _expand_clones(settings, partition, name, inventory, inventory_lock,
new_refs, moved_refs, deleted_refs):
bare_path = _compose_bare_repository_path(settings,
name, partition)
if new_refs:
logging.debug("Processing new refs of %s/%s (%s)..." % \
(partition, name, new_refs))
for refname in new_refs:
clone_path = _compose_clone_repository_path(settings,
name, partition, refname)
logging.info("Populating new ref '%s'" % clone_path)
try:
if ref_is_commit(settings, refname):
commit_id = refname.replace(settings.HASHPREFIX, '')
logging.debug("Will create a clone pointing to '%s'" % commit_id)
git.clone(clone_path, "%s" % bare_path, shared=True)
git.reset(clone_path, commit_id, hard=True)
else:
git.clone(clone_path, "%s" % bare_path, branch=refname)
# Needs reset so the proxy notices about the change on the mutable
# http://docs.python.org/2.7/library/multiprocessing.html#managers
# Locking on the assignment is guarateed by the library, but
# additional locking is needed as A = A + 1 is a critical section.
if inventory_lock:
inventory_lock.acquire()
inventory[name] += [refname]
if inventory_lock:
inventory_lock.release()
except JensGitError, error:
if os.path.isdir(clone_path):
shutil.rmtree(clone_path)
logging.error("Unable to create clone '%s' (%s)" % \
(clone_path, error))
def _read_desired_inventory():
desired = {'modules': {}, 'hostgroups': {}, 'common': {}}
environments = get_names_of_declared_environments()
for environmentname in environments:
try:
environment = read_environment_definition(environmentname)
# TODO: what if overrides empty? what if overrides,partition empty?
if 'overrides' in environment:
for partition in environment['overrides'].iterkeys():
if partition in ("modules", "hostgroups", "common"):
for name, override in \
environment['overrides'][partition].iteritems():
# prefixhash is equivalent to PREFIXhash, contrary to
# refs (branches, sic) which as case-sensitiive
if ref_is_commit(override):
override = override.lower()
if name not in desired[partition]:
desired[partition][name] = [override]
else:
if override not in desired[partition][name]:
desired[partition][name].append(override)
except JensEnvironmentsError, error:
logging.error("Unable to process '%s' definition. Skipping" %
environmentname)
continue # Just ignore, as won't be generated later on either.
def _compare_refs(old_refs, new_refs, inventory, desired):
settings = Settings()
desired = set(desired).union(settings.MANDATORY_BRANCHES)
# New: What we need minus what we have...
new = list(desired.difference(inventory))
# ...but only refs that exist or commits
new = [ref for ref in new if ref_is_commit(ref) or ref in new_refs]
# Deleted: what we have that we don't need anymore
deleted = list(set(inventory).difference(desired))
if new:
logging.debug("New refs to be expanded: %s", new)
if deleted:
logging.debug("Removed refs: %s", deleted)
# Candidates are those that we already have and we still need
moved = []
for ref in desired.intersection(inventory):
# No point in checking if a commit has moved
if ref_is_commit(ref):
continue
# If the ref is still being used (in the inventory and desired)
# but has been removed from the repo we mark it as delete.
# Next run will try to get it again and skip the expansion.
if ref not in new_refs:
logging.info("Ref '%s' still needed but removed from repo", ref)
deleted.append(ref)
continue
# The ref is still there and is gonna be kept, check if
# it has moved.
if new_refs[ref] != old_refs[ref]:
logging.debug("Ref '%s' has moved and points to %s",
ref, new_refs[ref])
moved.append(ref)
else:
logging.debug("Ref '%s' is known but didn't move", ref)
return new, moved, deleted
def _compare_refs(old_refs, new_refs, inventory, desired):
settings = Settings()
desired = set(desired).union(settings.MANDATORY_BRANCHES)
# New: What we need minus what we have...
new = list(desired.difference(inventory))
# ...but only refs that exist or commits
new = [ref for ref in new if ref_is_commit(ref) or ref in new_refs]
# Deleted: what we have that we don't need anymore
deleted = list(set(inventory).difference(desired))
if new:
logging.debug("New refs to be expanded: %s", new)
if deleted:
logging.debug("Removed refs: %s", deleted)
# Candidates are those that we already have and we still need
moved = []
for ref in desired.intersection(inventory):
# No point in checking if a commit has moved
if ref_is_commit(ref):
continue
# If the ref is still being used (in the inventory and desired)
# but has been removed from the repo we mark it as delete.
# Next run will try to get it again and skip the expansion.
if ref not in new_refs:
logging.info("Ref '%s' still needed but removed from repo", ref)
deleted.append(ref)
continue
# The ref is still there and is gonna be kept, check if
# it has moved.
if new_refs[ref] != old_refs[ref]:
logging.debug("Ref '%s' has moved and points to %s", ref,
new_refs[ref])
moved.append(ref)
else:
logging.debug("Ref '%s' is known but didn't move", ref)
return new, moved, deleted
def _create_new_repositories(new_repositories, partition,
definition, inventory, desired):
settings = Settings()
created = []
for repository in new_repositories:
logging.info("Cloning and expanding %s/%s...", partition, repository)
bare_path = _compose_bare_repository_path(repository, partition)
bare_url = definition['repositories'][partition][repository]
try:
git.clone(bare_path, bare_url, bare=True)
except JensGitError as error:
logging.error("Unable to clone '%s' (%s). Skipping.",
repository, error)
if os.path.exists(bare_path):
shutil.rmtree(bare_path)
continue
try:
refs = list(git.get_refs(bare_path).keys())
except JensGitError as error:
logging.error("Unable to get refs of '%s' (%s). Skipping.",
repository, error)
shutil.rmtree(bare_path)
logging.debug("Bare repository %s has been removed", bare_path)
continue
# Check if the repository has the mandatory branches
if all([ref in refs for ref in settings.MANDATORY_BRANCHES]):
# Expand only the mandatory and available requested branches
# commits will always be attempted to be expanded
new = set(settings.MANDATORY_BRANCHES)
new = new.union([ref for ref in desired.get(repository, [])
if ref_is_commit(ref) or ref in refs])
inventory[repository] = []
_expand_clones(partition, repository, inventory, None, new, [], [])
created.append(repository)
else:
logging.error("Repository '%s' lacks some of the mandatory branches. Skipping.",
repository)
shutil.rmtree(bare_path)
logging.debug("Bare repository %s has been removed", bare_path)
return created
continue
try:
refs = git.get_refs(bare_path).keys()
except JensGitError, error:
logging.error("Unable to get refs of '%s' (%s). Skipping." %
(repository, error))
shutil.rmtree(bare_path)
logging.debug("Bare repository %s has been removed" % bare_path)
continue
# Check if the repository has the mandatory branches
if all([ref in refs for ref in settings.MANDATORY_BRANCHES]):
# Expand only the mandatory and available requested branches
# commits will always be attempted to be expanded
new = set(settings.MANDATORY_BRANCHES)
new = new.union(
filter(lambda x: ref_is_commit(settings, x) or x in refs,
desired.get(repository, [])))
inventory[repository] = []
_expand_clones(settings, partition, repository, inventory, None,
new, [], [])
created.append(repository)
else:
logging.error(
"Repository '%s' lacks some of the mandatory branches. Skipping."
% repository)
shutil.rmtree(bare_path)
logging.debug("Bare repository %s has been removed" % bare_path)
return created
# This is the most common operation Jens has to do, git-fetch
if os.path.exists(bare_path):
shutil.rmtree(bare_path)
continue
try:
refs = git.get_refs(bare_path).keys()
except JensGitError, error:
logging.error("Unable to get refs of '%s' (%s). Skipping." % (repository, error))
shutil.rmtree(bare_path)
logging.debug("Bare repository %s has been removed" % bare_path)
continue
# Check if the repository has the mandatory branches
if all([ref in refs for ref in settings.MANDATORY_BRANCHES]):
# Expand only the mandatory and available requested branches
# commits will always be attempted to be expanded
new = set(settings.MANDATORY_BRANCHES)
new = new.union(filter(lambda x: ref_is_commit(settings, x) or x in refs,
desired.get(repository, [])))
inventory[repository] = []
_expand_clones(settings, partition, repository, inventory, None, new, [], [])
created.append(repository)
else:
logging.error("Repository '%s' lacks some of the mandatory branches. Skipping." %
repository)
shutil.rmtree(bare_path)
logging.debug("Bare repository %s has been removed" % bare_path)
return created
# This is the most common operation Jens has to do, git-fetch
# over all bare repos and the expansion of clones.
def _refresh_repositories(settings, existing_repositories, partition, inventory, desired):
if not existing_repositories:
def _expand_clones(partition, name, inventory, inventory_lock, new_refs,
moved_refs, deleted_refs):
settings = Settings()
bare_path = _compose_bare_repository_path(name, partition)
if new_refs:
logging.debug("Processing new refs of %s/%s (%s)...",
partition, name, new_refs)
for refname in new_refs:
clone_path = _compose_clone_repository_path(name, partition, refname)
logging.info("Populating new ref '%s'", clone_path)
try:
if ref_is_commit(refname):
commit_id = refname.replace(settings.HASHPREFIX, '')
logging.debug("Will create a clone pointing to '%s'", commit_id)
git.clone(clone_path, "%s" % bare_path, shared=True)
git.reset(clone_path, commit_id, hard=True)
else:
git.clone(clone_path, "%s" % bare_path, branch=refname)
# Needs reset so the proxy notices about the change on the mutable
# http://docs.python.org/2.7/library/multiprocessing.html#managers
# Locking on the assignment is guarateed by the library, but
# additional locking is needed as A = A + 1 is a critical section.
if inventory_lock:
inventory_lock.acquire()
inventory[name] += [refname]
if inventory_lock:
inventory_lock.release()
except JensGitError as error:
if os.path.isdir(clone_path):
shutil.rmtree(clone_path)
logging.error("Unable to create clone '%s' (%s)",
clone_path, error)
if moved_refs:
logging.debug("Processing moved refs of %s/%s (%s)...",
partition, name, moved_refs)
for refname in moved_refs:
clone_path = _compose_clone_repository_path(name, partition, refname)
logging.info("Updating ref '%s'", clone_path)
try:
# If this fails, the bare would have the correct HEADs
# but the clone will be out of date and won't ever be
# updated until a new commit arrives to the bare.
# Reason: a lock file left behind because Git was killed
# mid-flight.
git.fetch(clone_path)
git.reset(clone_path, "origin/%s" % refname, hard=True)
logging.info("Updated ref '%s' (%s)", clone_path,
git.get_head(clone_path, short=True))
except JensGitError as error:
logging.error("Unable to refresh clone '%s' (%s)",
clone_path, error)
if deleted_refs:
logging.debug("Processing deleted refs of %s/%s (%s)...",
partition, name, deleted_refs)
for refname in deleted_refs:
clone_path = _compose_clone_repository_path(name, partition, refname)
logging.info("Removing %s", clone_path)
try:
if os.path.isdir(clone_path):
shutil.rmtree(clone_path)
if refname in inventory[name]:
if inventory_lock:
inventory_lock.acquire()
element = inventory[name]
element.remove(refname)
inventory[name] = element
if inventory_lock:
inventory_lock.release()
logging.info("%s/%s deleted from inventory", name, refname)
except OSError as error:
logging.error("Couldn't delete %s/%s/%s (%s)",
partition, name, refname, error)
try:
refs = git.get_refs(bare_path).keys()
except JensGitError, error:
logging.error("Unable to get refs of '%s' (%s). Skipping.",
repository, error)
shutil.rmtree(bare_path)
logging.debug("Bare repository %s has been removed", bare_path)
continue
# Check if the repository has the mandatory branches
if all([ref in refs for ref in settings.MANDATORY_BRANCHES]):
# Expand only the mandatory and available requested branches
# commits will always be attempted to be expanded
new = set(settings.MANDATORY_BRANCHES)
new = new.union([
ref for ref in desired.get(repository, [])
if ref_is_commit(ref) or ref in refs
])
inventory[repository] = []
_expand_clones(partition, repository, inventory, None, new, [], [])
created.append(repository)
else:
logging.error(
"Repository '%s' lacks some of the mandatory branches. Skipping.",
repository)
shutil.rmtree(bare_path)
logging.debug("Bare repository %s has been removed", bare_path)
return created
# This is the most common operation Jens has to do, git-fetch
# over all bare repos and the expansion of clones.