def cli(ctx, todo_branch, repo): ctx.obj = obj = {} obj['todo_branch'] = todo_branch if repo is None: # walk upwards until we find a .git path path = os.path.abspath(os.getcwd()) while True: git_path = os.path.join(path, '.git') if os.path.exists(git_path) and os.path.isdir(git_path): repo = Repo(git_path) break path, tail = os.path.split(path) if not tail: break if repo is None: click.echo('No valid git repository found upwards of {}' .format(os.getcwd()), err=True) sys.exit(1) obj['repo'] = repo obj['gitconfig'] = gitconfig = repo.get_config_stack() obj['db'] = TODOBranch(repo, 'refs/heads/' + todo_branch) obj['user_name'] = gitconfig.get('user', 'name') obj['user_email'] = gitconfig.get('user', 'email') if ctx.invoked_subcommand is None: return ctx.invoke(list_todos)
def _fetch_remote_refs(cls, url: str, local: Repo) -> FetchPackResult: """ Helper method to fetch remote refs. """ client: GitClient path: str kwargs: dict[str, str] = {} credentials = get_default_authenticator().get_credentials_for_git_url( url=url) if credentials.password and credentials.username: # we do this conditionally as otherwise, dulwich might complain if these # parameters are passed in for an ssh url kwargs["username"] = credentials.username kwargs["password"] = credentials.password config = local.get_config_stack() client, path = get_transport_and_path(url, config=config, **kwargs) with local: result: FetchPackResult = client.fetch( path, local, determine_wants=local.object_store.determine_wants_all, ) return result
def configure(**kwargs): """ Parse configuration from git config """ config = {} # Get top level directory of project proc = subprocess.Popen(['git', 'rev-parse', '--show-toplevel'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) config['top_dir'] = proc.communicate()[0].strip() _repo = Repo(config['top_dir']) sc = _repo.get_config_stack() if proc.returncode != 0: # log.error("{0} :: {1}".format(__name__, exit_codes[exit_code])) raise GitDeployConfigError(message=exit_codes[20], exit_code=20) # Define the key names, git config names, and error codes config_elements = { 'hook_dir': ('deploy', 'hook-dir', 21), 'path': ('deploy', 'path', 23), 'user': ('deploy', 'user', 24), 'target': ('deploy', 'target', 25), 'repo_name': ('deploy', 'tag-prefix', 22), 'client_path': ('deploy', 'client-path', 19), 'user.name': ('user', 'name', 28), 'user.email': ('user', 'email', 29), 'deploy.key_path': ('deploy', 'key-path', 37), 'deploy.test_repo': ('deploy', 'test-repo-path', 38), 'deploy.remote_url': ('deploy', 'remote-url', 41), } # Assign the values of each git config element for key, value in config_elements.iteritems(): try: # Override with kwargs if the attribute exists if key in kwargs: config[key] = kwargs[key] else: config[key] = sc.get(value[0], value[1]) except KeyError as e: log.error("{0} :: {1}".format(__name__, e.message)) raise GitDeployConfigError(message=exit_codes[15], exit_code=15) config['sync_dir'] = '{0}/sync'.format(config['hook_dir']) config['deploy_root'] = config['client_path'] + '/.git/deploy' config['deploy_apps'] = config['deploy_root'] + '/apps' config['deploy_apps_common'] = config['deploy_root'] + '/apps/common' config['deploy_sync'] = config['deploy_root'] + '/sync' return config
import os from time import time from dulwich.repo import Repo from dulwich.objects import Blob, Tree, Commit _repo = Repo(".") _config = _repo.get_config_stack() def init(): try: _repo.refs["refs/heads/clask"] except KeyError: _commit({".gitkeep": ""}, "Initialize clask") def get(slug): return _get_blob(slug).data def all(): tree = _get_current_tree() return list(set(map(_slug, tree)) - set(["gitkeep"])) def put(slug, data, message=None, finish=False): tree_dict = dict() if finish: tree_dict[_tree_entry(slug, True)] = data
class GitRepo(object): def __init__(self, path): if os.path.exists(path): if not os.path.isdir(path): raise IOError('Git repository "%s" must be a directory.' % path) try: self.repo = Repo(path) except NotGitRepository: # repo does not exist self.repo = Repo.init(path, not os.path.exists(path)) self.temp_persist_files = [] def _get_commit(self, version="HEAD"): commit = self.repo[version] if not isinstance(commit, Commit): raise NotCommitError(commit) return commit def get_type(self, name, version="HEAD"): commit = self._get_commit(version) tree = self.repo.tree(commit.tree) if name not in tree: raise KeyError('Cannot find object "%s"' % name) if tree[name][0] & stat.S_IFDIR: return "tree" else: return "blob" def get_path(self, name, version="HEAD", path_type=None, out_name=None, out_suffix=''): if path_type is None: path_type = self.get_type(name, version) if path_type == 'tree': return self.get_dir(name, version, out_name, out_suffix) elif path_type == 'blob': return self.get_file(name, version, out_name, out_suffix) raise TypeError("Unknown path type '%s'" % path_type) def _write_blob(self, blob_sha, out_fname=None, out_suffix=''): if out_fname is None: # create a temporary file (fd, out_fname) = tempfile.mkstemp(suffix=out_suffix, prefix='vt_persist') os.close(fd) self.temp_persist_files.append(out_fname) else: out_dirname = os.path.dirname(out_fname) if out_dirname and not os.path.exists(out_dirname): os.makedirs(out_dirname) blob = self.repo.get_blob(blob_sha) with open(out_fname, "wb") as f: for b in blob.as_raw_chunks(): f.write(b) return out_fname def get_file(self, name, version="HEAD", out_fname=None, out_suffix=''): commit = self._get_commit(version) tree = self.repo.tree(commit.tree) if name not in tree: raise KeyError('Cannot find blob "%s"' % name) blob_sha = tree[name][1] out_fname = self._write_blob(blob_sha, out_fname, out_suffix) return out_fname def get_dir(self, name, version="HEAD", out_dirname=None, out_suffix=''): if out_dirname is None: # create a temporary directory out_dirname = tempfile.mkdtemp(suffix=out_suffix, prefix='vt_persist') self.temp_persist_files.append(out_dirname) elif not os.path.exists(out_dirname): os.makedirs(out_dirname) commit = self._get_commit(version) tree = self.repo.tree(commit.tree) if name not in tree: raise KeyError('Cannot find tree "%s"' % name) subtree_id = tree[name][1] # subtree = self.repo.tree(subtree_id) for entry in self.repo.object_store.iter_tree_contents(subtree_id): out_fname = os.path.join(out_dirname, entry.path) self._write_blob(entry.sha, out_fname) return out_dirname def get_hash(self, name, version="HEAD", path_type=None): commit = self._get_commit(version) tree = self.repo.tree(commit.tree) if name not in tree: raise KeyError('Cannot find object "%s"' % name) return tree[name][1] @staticmethod def compute_blob_hash(fname, chunk_size=1 << 16): obj_len = os.path.getsize(fname) head = object_header(Blob.type_num, obj_len) with open(fname, "rb") as f: def read_chunk(): return f.read(chunk_size) my_iter = chain([head], iter(read_chunk, '')) return iter_sha1(my_iter) @staticmethod def compute_tree_hash(dirname): tree = Tree() for entry in sorted(os.listdir(dirname)): fname = os.path.join(dirname, entry) if os.path.isdir(fname): thash = GitRepo.compute_tree_hash(fname) mode = stat.S_IFDIR # os.stat(fname)[stat.ST_MODE] tree.add(entry, mode, thash) elif os.path.isfile(fname): bhash = GitRepo.compute_blob_hash(fname) mode = os.stat(fname)[stat.ST_MODE] tree.add(entry, mode, bhash) return tree.id @staticmethod def compute_hash(path): if os.path.isdir(path): return GitRepo.compute_tree_hash(path) elif os.path.isfile(path): return GitRepo.compute_blob_hash(path) raise TypeError("Do not support this type of path") def get_latest_version(self, path): head = self.repo.head() walker = Walker(self.repo.object_store, [head], max_entries=1, paths=[path]) return iter(walker).next().commit.id def _stage(self, filename): fullpath = os.path.join(self.repo.path, filename) if os.path.islink(fullpath): debug.warning("Warning: not staging symbolic link %s" % os.path.basename(filename)) elif os.path.isdir(fullpath): for f in os.listdir(fullpath): self._stage(os.path.join(filename, f)) else: if os.path.sep != '/': filename = filename.replace(os.path.sep, '/') self.repo.stage(filename) def add_commit(self, filename): self.setup_git() self._stage(filename) commit_id = self.repo.do_commit('Updated %s' % filename) return commit_id def setup_git(self): config_stack = self.repo.get_config_stack() try: config_stack.get(('user', ), 'name') config_stack.get(('user', ), 'email') except KeyError: from vistrails.core.system import current_user from dulwich.config import ConfigFile user = current_user() repo_conf = self.repo.get_config() repo_conf.set(('user', ), 'name', user) repo_conf.set(('user', ), 'email', '%s@localhost' % user) repo_conf.write_to_path()
class GitRepo(object): def __init__(self, path): if os.path.exists(path): if not os.path.isdir(path): raise IOError('Git repository "%s" must be a directory.' % path) try: self.repo = Repo(path) except NotGitRepository: # repo does not exist self.repo = Repo.init(path, not os.path.exists(path)) self.temp_persist_files = [] def _get_commit(self, version="HEAD"): commit = self.repo[version] if not isinstance(commit, Commit): raise NotCommitError(commit) return commit def get_type(self, name, version="HEAD"): commit = self._get_commit(version) tree = self.repo.tree(commit.tree) if name not in tree: raise KeyError('Cannot find object "%s"' % name) if tree[name][0] & stat.S_IFDIR: return "tree" else: return "blob" def get_path(self, name, version="HEAD", path_type=None, out_name=None, out_suffix=''): if path_type is None: path_type = self.get_type(name, version) if path_type == 'tree': return self.get_dir(name, version, out_name, out_suffix) elif path_type == 'blob': return self.get_file(name, version, out_name, out_suffix) raise TypeError("Unknown path type '%s'" % path_type) def _write_blob(self, blob_sha, out_fname=None, out_suffix=''): if out_fname is None: # create a temporary file (fd, out_fname) = tempfile.mkstemp(suffix=out_suffix, prefix='vt_persist') os.close(fd) self.temp_persist_files.append(out_fname) else: out_dirname = os.path.dirname(out_fname) if out_dirname and not os.path.exists(out_dirname): os.makedirs(out_dirname) blob = self.repo.get_blob(blob_sha) with open(out_fname, "wb") as f: for b in blob.as_raw_chunks(): f.write(b) return out_fname def get_file(self, name, version="HEAD", out_fname=None, out_suffix=''): commit = self._get_commit(version) tree = self.repo.tree(commit.tree) if name not in tree: raise KeyError('Cannot find blob "%s"' % name) blob_sha = tree[name][1] out_fname = self._write_blob(blob_sha, out_fname, out_suffix) return out_fname def get_dir(self, name, version="HEAD", out_dirname=None, out_suffix=''): if out_dirname is None: # create a temporary directory out_dirname = tempfile.mkdtemp(suffix=out_suffix, prefix='vt_persist') self.temp_persist_files.append(out_dirname) elif not os.path.exists(out_dirname): os.makedirs(out_dirname) commit = self._get_commit(version) tree = self.repo.tree(commit.tree) if name not in tree: raise KeyError('Cannot find tree "%s"' % name) subtree_id = tree[name][1] # subtree = self.repo.tree(subtree_id) for entry in self.repo.object_store.iter_tree_contents(subtree_id): out_fname = os.path.join(out_dirname, entry.path) self._write_blob(entry.sha, out_fname) return out_dirname def get_hash(self, name, version="HEAD", path_type=None): commit = self._get_commit(version) tree = self.repo.tree(commit.tree) if name not in tree: raise KeyError('Cannot find object "%s"' % name) return tree[name][1] @staticmethod def compute_blob_hash(fname, chunk_size=1<<16): obj_len = os.path.getsize(fname) head = object_header(Blob.type_num, obj_len) with open(fname, "rb") as f: def read_chunk(): return f.read(chunk_size) my_iter = chain([head], iter(read_chunk,'')) return iter_sha1(my_iter) return None @staticmethod def compute_tree_hash(dirname): tree = Tree() for entry in sorted(os.listdir(dirname)): fname = os.path.join(dirname, entry) if os.path.isdir(fname): thash = GitRepo.compute_tree_hash(fname) mode = stat.S_IFDIR # os.stat(fname)[stat.ST_MODE] tree.add(entry, mode, thash) elif os.path.isfile(fname): bhash = GitRepo.compute_blob_hash(fname) mode = os.stat(fname)[stat.ST_MODE] tree.add(entry, mode, bhash) return tree.id @staticmethod def compute_hash(path): if os.path.isdir(path): return GitRepo.compute_tree_hash(path) elif os.path.isfile(path): return GitRepo.compute_blob_hash(path) raise TypeError("Do not support this type of path") def get_latest_version(self, path): head = self.repo.head() walker = Walker(self.repo.object_store, [head], max_entries=1, paths=[path]) return iter(walker).next().commit.id def _stage(self, filename): fullpath = os.path.join(self.repo.path, filename) if os.path.islink(fullpath): debug.warning("Warning: not staging symbolic link %s" % os.path.basename(filename)) elif os.path.isdir(fullpath): for f in os.listdir(fullpath): self._stage(os.path.join(filename, f)) else: if os.path.sep != '/': filename = filename.replace(os.path.sep, '/') self.repo.stage(filename) def add_commit(self, filename): self.setup_git() self._stage(filename) commit_id = self.repo.do_commit('Updated %s' % filename) return commit_id def setup_git(self): config_stack = self.repo.get_config_stack() try: config_stack.get(('user',), 'name') config_stack.get(('user',), 'email') except KeyError: from vistrails.core.system import current_user from dulwich.config import ConfigFile user = current_user() repo_conf = self.repo.get_config() repo_conf.set(('user',), 'name', user) repo_conf.set(('user',), 'email', '%s@localhost' % user) repo_conf.write_to_path()
def main(): import argparse import sys from logbook.more import ColorizedStderrHandler from logbook.handlers import NullHandler from . import __version__ default_footer = '\n\n(commit by unleash %s)' % __version__ parser = argparse.ArgumentParser() sub = parser.add_subparsers(dest='action') parser.add_argument('-r', '--root', default=os.getcwd(), type=os.path.abspath, help='Root directory for package.') parser.add_argument('-a', '--author', default=None, help='Author string for commits (uses git configured ' 'settings per default') parser.add_argument('-b', '--batch', default=True, dest='interactive', action='store_true', help='Do not ask for confirmation before committing ' 'changes to anything.') parser.add_argument('-d', '--debug', default=logbook.INFO, dest='loglevel', action='store_const', const=logbook.DEBUG) parser.add_argument('--version', action='version', version='%(prog)s ' + __version__) create_release = sub.add_parser('create-release') create_release.add_argument('-b', '--branch', default='master') create_release.add_argument('-v', '--release-version', default=None) create_release.add_argument('-d', '--dev-version', default=None) create_release.add_argument('-F', '--no-footer', default=default_footer, dest='commit_footer', action='store_const', const='', help='Do not output footer on commit messages.' ) create_release.add_argument('-n', '--package-name', default=None, help='The name of the package to be packaged.') publish_release = sub.add_parser('publish') publish_release.add_argument('-s', '--sign') publish_release.add_argument('-v', '--version', default=None, help=('Name of the tag to publish. Defaults ' 'to the tag whose commit has the ' 'highest readable version.')) args = parser.parse_args() NullHandler().push_application() ColorizedStderrHandler(format_string='{record.message}', level=args.loglevel).push_application() # first, determine current version repo = Repo(args.root) config = repo.get_config_stack() if args.author is None: args.author = '%s <%s>' % ( config.get('user', 'name'), config.get('user', 'email') ) func = globals()['action_' + args.action.replace('-', '_')] try: return func(args=args, repo=repo) except Exception as e: log.error(str(e)) if args.loglevel == logbook.DEBUG: log.exception(e) sys.exit(1)
def main(): args = parser.parse_args() # open repo repo = Repo(args.repo) config = repo.get_config_stack() ref_name = b'refs/heads/' + args.branch.encode('ascii') old_head = repo.refs[ref_name] if ref_name in repo.refs else None root = OrderedDict() for orig, s_args, s_kwargs in args.sources: scheme = s_args.pop(0) # prepare include/exclude expressions include_exprs = [fnmatch.translate(pattern) for pattern in s_kwargs.pop('include', [])] include_exprs.extend(s_kwargs.pop('rinclude', [])) exclude_exprs = [fnmatch.translate(pattern) for pattern in s_kwargs.pop('exclude', [])] exclude_exprs.extend(s_kwargs.pop('rexclude', [])) includes = list(map(re.compile, include_exprs)) excludes = list(map(re.compile, exclude_exprs)) srcs = list(SOURCES[scheme].create(*s_args, **s_kwargs)) sys.stderr.write(orig) sys.stderr.write('\n') for src in srcs: for path in src: # if includes are specified and none matches, skip if includes and not filter(lambda exp: exp.match(path), includes): continue # vice-versa for excludes if excludes and filter(lambda exp: exp.match(path), excludes): continue # add the blob mode, blob = src.get_blob(path) repo.object_store.add_object(blob) # tree entry node = root components = path.split('/') for c in components[:-1]: node = node.setdefault(c, OrderedDict()) node[components[-1]] = (mode, blob.id) sys.stderr.write(path) sys.stderr.write('\n') sys.stderr.write('\n') # collect trees def store_tree(node): tree = Tree() for name in node: if isinstance(node[name], dict): tree.add(name.encode(args.encoding), MODE_TREE, store_tree(node[name]).id) else: tree.add(name.encode(args.encoding), *node[name]) repo.object_store.add_object(tree) return tree tree = store_tree(root) def get_user(): name = config.get(b'user', b'name').decode('utf8') email = config.get(b'user', b'email').decode('utf8') return '{} <{}>'.format(name, email) commit = Commit() if old_head: commit.parents = [old_head] commit.tree = tree.id commit.author = (args.author or get_user()).encode(args.encoding) commit.committer = (args.committer or get_user()).encode(args.encoding) commit.commit_time = args.commit_time commit.author_time = args.author_time commit.commit_timezone = args.commit_timezone[0] commit.author_timezone = args.author_timezone[0] commit.encoding = args.encoding.encode('ascii') commit.message = args.message.encode(args.encoding) repo.object_store.add_object(commit) # set ref if old_head: repo.refs.set_if_equals(ref_name, old_head, commit.id) else: repo.refs.add_if_new(ref_name, commit.id) print(commit.id)
class Unleash(object): def _create_child_commit(self, parent_ref): parent = ResolvedRef(self.repo, parent_ref) if not parent.is_definite: raise InvocationError('{} is ambiguous: {}'.format( parent.ref, parent.full_name )) if not parent.found: raise InvocationError('Could not resolve "{}"'.format(parent.ref)) # prepare the release commit commit = MalleableCommit.from_existing( self.repo, parent.id ) # update author and such if opts['author'] is None: commit.author = '{} <{}>'.format( self.gitconfig.get('user', 'name'), self.gitconfig.get('user', 'email'), ) commit.commiter = commit.author else: commit.author = opts['author'] commit.committer = opts['author'] now = int(time.time()) ltz = get_local_timezone(now) commit.author_time = now commit.author_timezone = ltz commit.commit_time = now commit.commit_timezone = ltz commit.parent_ids = [parent.id] return commit def __init__(self, plugins=[]): self.plugins = plugins def _init_repo(self): self.repo = Repo(opts['root']) self.gitconfig = self.repo.get_config_stack() def _perform_step(self, signal_name): log.debug('begin: {}'.format(signal_name)) begin = time.time() # create new top-level context with new_local_stack() as nc: nc['issues'] = issues.channel(signal_name) self.plugins.notify(signal_name) duration = time.time() - begin log.debug('end: {}, took {:.4f}s'.format(signal_name, duration)) def create_release(self, ref): with new_local_stack() as nc: # resolve reference base_ref = ResolvedRef(self.repo, ref) log.debug( 'Base ref: {} ({})'.format(base_ref.full_name, base_ref.id) ) orig_tree = base_ref.get_object().tree # initialize context nc['commit'] = self._create_child_commit(ref) nc['issues'] = IssueCollector(log=log) nc['info'] = {'ref': base_ref} nc['log'] = log try: self._perform_step('collect_info') log.debug('info: {}'.format(pformat(info))) self._perform_step('prepare_release') self._perform_step('lint_release') if opts['inspect']: log.info(unicode(commit)) # check out to temporary directory with TempDir() as inspect_dir: commit.export_to(inspect_dir) log.info( 'You are being dropped into an interactive shell ' 'inside a temporary checkout of the release ' 'commit. No changes you make will persist. Exit ' 'the shell to abort the release process.\n\n' 'Use "exit 2" to continue the release.' ) status = run_user_shell(cwd=inspect_dir) if status != 2: raise InvocationError( 'Aborting release, got exit code {} from shell.'. format(status)) # save release commit release_commit = nc['commit'] # we're done with the release, now create the dev commit nc['commit'] = self._create_child_commit(ref) nc['issues'] = IssueCollector(log=log) # creating development commit self._perform_step('prepare_dev') if opts['dry_run']: log.info('Not saving created commits. Dry-run successful.') return # we've got both commits, now tag the release confirm_prompt( 'Advance dev to {} and release {}?' .format(info['dev_version'], info['release_version']) ) release_tag = 'refs/tags/{}'.format(info['release_version']) if release_tag in self.repo.refs: confirm_prompt( 'Repository already contains {}, really overwrite tag?' .format(release_tag), ) release_hash = release_commit.save() log.info('{}: {}'.format(release_tag, release_hash)) self.repo.refs[release_tag] = release_hash # save the dev commit dev_hash = nc['commit'].save() # if our release commit formed from a branch, we set that branch # to our new dev commit assert base_ref.is_definite and base_ref.found if not base_ref.is_ref or\ not base_ref.full_name.startswith('refs/heads'): log.warning('Release commit does not originate from a ' 'branch; dev commit will not be reachable.') log.info('Dev commit: {}'.format(dev_hash)) else: self.repo.refs[base_ref.full_name] = dev_hash # change the branch to point at our new dev commit log.info('{}: {}'.format( base_ref.full_name, dev_hash )) self._update_working_copy(base_ref, orig_tree) except PluginError: # just abort, error has been logged already log.debug('Exiting due to PluginError') return def _update_working_copy(self, base_ref, orig_tree): head_ref = ResolvedRef(self.repo, 'HEAD') if not head_ref.is_definite or not head_ref.is_symbolic\ or not head_ref.target == base_ref.full_name: log.info('HEAD is not a symbolic ref to {}, leaving your ' 'working copy untouched.') return if not self.repo.has_index(): log.info('Repository has no index, not updating working copy.') return index = self.repo.open_index() changes = list(index.changes_from_tree( self.repo.object_store, orig_tree, )) if changes: log.warning('There are staged changes in your index. Will not ' 'update working copy.\n\n' 'You will need to manually change your HEAD to ' '{}.'.format(base_ref.id)) return # reset the index to the new dev commit confirm_prompt( 'Do you want to reset your index to the new dev commit and check ' 'it out? Unsaved changes to your working copy may be overwritten!' ) log.info('Resetting index and checking out dev commit.') build_index_from_tree( self.repo.path, self.repo.index_path(), self.repo.object_store, base_ref.get_object().tree, ) def publish(self, ref): if ref is None: tags = sorted( (t for t in self.repo.refs.as_dict().iteritems() if t[0].startswith('refs/tags')), key=lambda (_, sha): self.repo[sha].commit_time, reverse=True, ) if not tags: log.error('Could not find a tag to publish.') return ref = tags[0][0] pref = ResolvedRef(self.repo, ref) with new_local_stack() as nc: nc['commit'] = MalleableCommit.from_existing(self.repo, pref.id) log.debug('Release tag: {}'.format(commit)) nc['issues'] = IssueCollector(log=log) nc['info'] = {'ref': pref} nc['log'] = log try: self._perform_step('collect_info') log.debug('info: {}'.format(pformat(info))) self._perform_step('publish_release') except PluginError: log.debug('Exiting due to PluginError') return