def _read_desired_inventory(settings): desired = {'modules': {}, 'hostgroups': {}, 'common': {}} environments = get_names_of_declared_environments(settings) for environmentname in environments: try: environment = read_environment_definition( settings, environmentname) # TODO: what if overrides empty? what if overrides,partition empty? if 'overrides' in environment: for partition in environment['overrides'].iterkeys(): if partition in ("modules", "hostgroups", "common"): for name, override in \ environment['overrides'][partition].iteritems(): # prefixhash is equivalent to PREFIXhash, contrary to # refs (branches, sic) which as case-sensitiive if ref_is_commit(settings, override): override = override.lower() if name not in desired[partition]: desired[partition][name] = [override] else: if override not in desired[partition][name]: desired[partition][name].append(override) except JensEnvironmentsError, error: logging.error("Unable to process '%s' definition. Skipping" % \ environmentname) continue # Just ignore, as won't be generated later on either.
def _expand_clones(settings, partition, name, inventory, inventory_lock, new_refs, moved_refs, deleted_refs): bare_path = _compose_bare_repository_path(settings, name, partition) if new_refs: logging.debug("Processing new refs of %s/%s (%s)..." % \ (partition, name, new_refs)) for refname in new_refs: clone_path = _compose_clone_repository_path(settings, name, partition, refname) logging.info("Populating new ref '%s'" % clone_path) try: if ref_is_commit(settings, refname): commit_id = refname.replace(settings.HASHPREFIX, '') logging.debug("Will create a clone pointing to '%s'" % commit_id) git.clone(clone_path, "%s" % bare_path, shared=True) git.reset(clone_path, commit_id, hard=True) else: git.clone(clone_path, "%s" % bare_path, branch=refname) # Needs reset so the proxy notices about the change on the mutable # http://docs.python.org/2.7/library/multiprocessing.html#managers # Locking on the assignment is guarateed by the library, but # additional locking is needed as A = A + 1 is a critical section. if inventory_lock: inventory_lock.acquire() inventory[name] += [refname] if inventory_lock: inventory_lock.release() except JensGitError, error: if os.path.isdir(clone_path): shutil.rmtree(clone_path) logging.error("Unable to create clone '%s' (%s)" % \ (clone_path, error))
def _read_desired_inventory(): desired = {'modules': {}, 'hostgroups': {}, 'common': {}} environments = get_names_of_declared_environments() for environmentname in environments: try: environment = read_environment_definition(environmentname) # TODO: what if overrides empty? what if overrides,partition empty? if 'overrides' in environment: for partition in environment['overrides'].iterkeys(): if partition in ("modules", "hostgroups", "common"): for name, override in \ environment['overrides'][partition].iteritems(): # prefixhash is equivalent to PREFIXhash, contrary to # refs (branches, sic) which as case-sensitiive if ref_is_commit(override): override = override.lower() if name not in desired[partition]: desired[partition][name] = [override] else: if override not in desired[partition][name]: desired[partition][name].append(override) except JensEnvironmentsError, error: logging.error("Unable to process '%s' definition. Skipping" % environmentname) continue # Just ignore, as won't be generated later on either.
def _compare_refs(old_refs, new_refs, inventory, desired): settings = Settings() desired = set(desired).union(settings.MANDATORY_BRANCHES) # New: What we need minus what we have... new = list(desired.difference(inventory)) # ...but only refs that exist or commits new = [ref for ref in new if ref_is_commit(ref) or ref in new_refs] # Deleted: what we have that we don't need anymore deleted = list(set(inventory).difference(desired)) if new: logging.debug("New refs to be expanded: %s", new) if deleted: logging.debug("Removed refs: %s", deleted) # Candidates are those that we already have and we still need moved = [] for ref in desired.intersection(inventory): # No point in checking if a commit has moved if ref_is_commit(ref): continue # If the ref is still being used (in the inventory and desired) # but has been removed from the repo we mark it as delete. # Next run will try to get it again and skip the expansion. if ref not in new_refs: logging.info("Ref '%s' still needed but removed from repo", ref) deleted.append(ref) continue # The ref is still there and is gonna be kept, check if # it has moved. if new_refs[ref] != old_refs[ref]: logging.debug("Ref '%s' has moved and points to %s", ref, new_refs[ref]) moved.append(ref) else: logging.debug("Ref '%s' is known but didn't move", ref) return new, moved, deleted
def _create_new_repositories(new_repositories, partition, definition, inventory, desired): settings = Settings() created = [] for repository in new_repositories: logging.info("Cloning and expanding %s/%s...", partition, repository) bare_path = _compose_bare_repository_path(repository, partition) bare_url = definition['repositories'][partition][repository] try: git.clone(bare_path, bare_url, bare=True) except JensGitError as error: logging.error("Unable to clone '%s' (%s). Skipping.", repository, error) if os.path.exists(bare_path): shutil.rmtree(bare_path) continue try: refs = list(git.get_refs(bare_path).keys()) except JensGitError as error: logging.error("Unable to get refs of '%s' (%s). Skipping.", repository, error) shutil.rmtree(bare_path) logging.debug("Bare repository %s has been removed", bare_path) continue # Check if the repository has the mandatory branches if all([ref in refs for ref in settings.MANDATORY_BRANCHES]): # Expand only the mandatory and available requested branches # commits will always be attempted to be expanded new = set(settings.MANDATORY_BRANCHES) new = new.union([ref for ref in desired.get(repository, []) if ref_is_commit(ref) or ref in refs]) inventory[repository] = [] _expand_clones(partition, repository, inventory, None, new, [], []) created.append(repository) else: logging.error("Repository '%s' lacks some of the mandatory branches. Skipping.", repository) shutil.rmtree(bare_path) logging.debug("Bare repository %s has been removed", bare_path) return created
continue try: refs = git.get_refs(bare_path).keys() except JensGitError, error: logging.error("Unable to get refs of '%s' (%s). Skipping." % (repository, error)) shutil.rmtree(bare_path) logging.debug("Bare repository %s has been removed" % bare_path) continue # Check if the repository has the mandatory branches if all([ref in refs for ref in settings.MANDATORY_BRANCHES]): # Expand only the mandatory and available requested branches # commits will always be attempted to be expanded new = set(settings.MANDATORY_BRANCHES) new = new.union( filter(lambda x: ref_is_commit(settings, x) or x in refs, desired.get(repository, []))) inventory[repository] = [] _expand_clones(settings, partition, repository, inventory, None, new, [], []) created.append(repository) else: logging.error( "Repository '%s' lacks some of the mandatory branches. Skipping." % repository) shutil.rmtree(bare_path) logging.debug("Bare repository %s has been removed" % bare_path) return created # This is the most common operation Jens has to do, git-fetch
if os.path.exists(bare_path): shutil.rmtree(bare_path) continue try: refs = git.get_refs(bare_path).keys() except JensGitError, error: logging.error("Unable to get refs of '%s' (%s). Skipping." % (repository, error)) shutil.rmtree(bare_path) logging.debug("Bare repository %s has been removed" % bare_path) continue # Check if the repository has the mandatory branches if all([ref in refs for ref in settings.MANDATORY_BRANCHES]): # Expand only the mandatory and available requested branches # commits will always be attempted to be expanded new = set(settings.MANDATORY_BRANCHES) new = new.union(filter(lambda x: ref_is_commit(settings, x) or x in refs, desired.get(repository, []))) inventory[repository] = [] _expand_clones(settings, partition, repository, inventory, None, new, [], []) created.append(repository) else: logging.error("Repository '%s' lacks some of the mandatory branches. Skipping." % repository) shutil.rmtree(bare_path) logging.debug("Bare repository %s has been removed" % bare_path) return created # This is the most common operation Jens has to do, git-fetch # over all bare repos and the expansion of clones. def _refresh_repositories(settings, existing_repositories, partition, inventory, desired): if not existing_repositories:
def _expand_clones(partition, name, inventory, inventory_lock, new_refs, moved_refs, deleted_refs): settings = Settings() bare_path = _compose_bare_repository_path(name, partition) if new_refs: logging.debug("Processing new refs of %s/%s (%s)...", partition, name, new_refs) for refname in new_refs: clone_path = _compose_clone_repository_path(name, partition, refname) logging.info("Populating new ref '%s'", clone_path) try: if ref_is_commit(refname): commit_id = refname.replace(settings.HASHPREFIX, '') logging.debug("Will create a clone pointing to '%s'", commit_id) git.clone(clone_path, "%s" % bare_path, shared=True) git.reset(clone_path, commit_id, hard=True) else: git.clone(clone_path, "%s" % bare_path, branch=refname) # Needs reset so the proxy notices about the change on the mutable # http://docs.python.org/2.7/library/multiprocessing.html#managers # Locking on the assignment is guarateed by the library, but # additional locking is needed as A = A + 1 is a critical section. if inventory_lock: inventory_lock.acquire() inventory[name] += [refname] if inventory_lock: inventory_lock.release() except JensGitError as error: if os.path.isdir(clone_path): shutil.rmtree(clone_path) logging.error("Unable to create clone '%s' (%s)", clone_path, error) if moved_refs: logging.debug("Processing moved refs of %s/%s (%s)...", partition, name, moved_refs) for refname in moved_refs: clone_path = _compose_clone_repository_path(name, partition, refname) logging.info("Updating ref '%s'", clone_path) try: # If this fails, the bare would have the correct HEADs # but the clone will be out of date and won't ever be # updated until a new commit arrives to the bare. # Reason: a lock file left behind because Git was killed # mid-flight. git.fetch(clone_path) git.reset(clone_path, "origin/%s" % refname, hard=True) logging.info("Updated ref '%s' (%s)", clone_path, git.get_head(clone_path, short=True)) except JensGitError as error: logging.error("Unable to refresh clone '%s' (%s)", clone_path, error) if deleted_refs: logging.debug("Processing deleted refs of %s/%s (%s)...", partition, name, deleted_refs) for refname in deleted_refs: clone_path = _compose_clone_repository_path(name, partition, refname) logging.info("Removing %s", clone_path) try: if os.path.isdir(clone_path): shutil.rmtree(clone_path) if refname in inventory[name]: if inventory_lock: inventory_lock.acquire() element = inventory[name] element.remove(refname) inventory[name] = element if inventory_lock: inventory_lock.release() logging.info("%s/%s deleted from inventory", name, refname) except OSError as error: logging.error("Couldn't delete %s/%s/%s (%s)", partition, name, refname, error)
try: refs = git.get_refs(bare_path).keys() except JensGitError, error: logging.error("Unable to get refs of '%s' (%s). Skipping.", repository, error) shutil.rmtree(bare_path) logging.debug("Bare repository %s has been removed", bare_path) continue # Check if the repository has the mandatory branches if all([ref in refs for ref in settings.MANDATORY_BRANCHES]): # Expand only the mandatory and available requested branches # commits will always be attempted to be expanded new = set(settings.MANDATORY_BRANCHES) new = new.union([ ref for ref in desired.get(repository, []) if ref_is_commit(ref) or ref in refs ]) inventory[repository] = [] _expand_clones(partition, repository, inventory, None, new, [], []) created.append(repository) else: logging.error( "Repository '%s' lacks some of the mandatory branches. Skipping.", repository) shutil.rmtree(bare_path) logging.debug("Bare repository %s has been removed", bare_path) return created # This is the most common operation Jens has to do, git-fetch # over all bare repos and the expansion of clones.