def init_test_repo(): """ Create a test repo, change to directory """ mkdir(config.TEST_REPO) Repo.init(config.TEST_REPO) chdir(config.TEST_REPO)
def cli(ctx, todo_branch, repo): ctx.obj = obj = {} obj['todo_branch'] = todo_branch if repo is None: # walk upwards until we find a .git path path = os.path.abspath(os.getcwd()) while True: git_path = os.path.join(path, '.git') if os.path.exists(git_path) and os.path.isdir(git_path): repo = Repo(git_path) break path, tail = os.path.split(path) if not tail: break if repo is None: click.echo('No valid git repository found upwards of {}' .format(os.getcwd()), err=True) sys.exit(1) obj['repo'] = repo obj['gitconfig'] = gitconfig = repo.get_config_stack() obj['db'] = TODOBranch(repo, 'refs/heads/' + todo_branch) obj['user_name'] = gitconfig.get('user', 'name') obj['user_email'] = gitconfig.get('user', 'email') if ctx.invoked_subcommand is None: return ctx.invoke(list_todos)
def writefile(namespacepath, path, data): """ Writes data to a file. @param fullpath: fullpath to a file @return: True or False """ fullpath = "%s/%s" % (namespacepath, path) # Write the data to the file try: f = open(fullpath, 'w') f.write(data) f.close() except: return (False, "Could not write file %s" % fullpath) # Now add it to git. try: repo = Repo(namespacepath) repo.stage(path) # Obviously, we'll want to get this commit info from somewhere else. commit_id = repo.do_commit( "An API commit", committer="API Committer <*****@*****.**>") except: return (False, "Could not commit file %s to namespace %s" % (path, namespace)) return (True, "Commited as %s" % commit_id)
def __init__(self, repo_or_path, origin_uri=None, auth=None, report_activity=None, *args, **kwargs): if isinstance(repo_or_path, DulwichRepo): self.repo = repo_or_path elif isinstance(repo_or_path, Gittle): self.repo = DulwichRepo(repo_or_path.path) elif isinstance(repo_or_path, basestring): path = os.path.abspath(repo_or_path) self.repo = DulwichRepo(path) else: logging.warning("Repo is of type %s" % type(repo_or_path)) raise Exception("Gittle must be initialized with either a dulwich repository or a string to the path") # Set path self.path = self.repo.path # The remote url self.origin_uri = origin_uri # Report client activty self._report_activity = report_activity # Build ignore filter self.hidden_regexes = copy.copy(self.HIDDEN_REGEXES) self.hidden_regexes.extend(self._get_ignore_regexes()) self.ignore_filter = utils.paths.path_filter_regex(self.hidden_regexes) self.filters = [self.ignore_filter] # Get authenticator if auth: self.authenticator = auth else: self.auth(*args, **kwargs)
def clone(source, target=None, bare=False, outstream=sys.stdout): """Clone a local or remote git repository. :param source: Path or URL for source repository :param target: Path to target repository (optional) :param bare: Whether or not to create a bare repository :param outstream: Optional stream to write progress to :return: The new repository """ client, host_path = get_transport_and_path(source) if target is None: target = host_path.split("/")[-1] if not os.path.exists(target): os.mkdir(target) if bare: r = Repo.init_bare(target) else: r = Repo.init(target) remote_refs = client.fetch(host_path, r, determine_wants=r.object_store.determine_wants_all, progress=outstream.write) r["HEAD"] = remote_refs["HEAD"] return r
def clone(source, target=None, bare=False, checkout=None, outstream=sys.stdout): """Clone a local or remote git repository. :param source: Path or URL for source repository :param target: Path to target repository (optional) :param bare: Whether or not to create a bare repository :param outstream: Optional stream to write progress to :return: The new repository """ if checkout is None: checkout = (not bare) if checkout and bare: raise ValueError("checkout and bare are incompatible") client, host_path = get_transport_and_path(source) if target is None: target = host_path.split("/")[-1] if not os.path.exists(target): os.mkdir(target) if bare: r = Repo.init_bare(target) else: r = Repo.init(target) remote_refs = client.fetch(host_path, r, determine_wants=r.object_store.determine_wants_all, progress=outstream.write) r["HEAD"] = remote_refs["HEAD"] if checkout: outstream.write('Checking out HEAD') index.build_index_from_tree(r.path, r.index_path(), r.object_store, r["HEAD"].tree) return r
def test_simple_local(self): f1_1 = make_object(Blob, data=b'f1') commit_spec = [[1], [2, 1], [3, 1, 2]] trees = {1: [(b'f1', f1_1), (b'f2', f1_1)], 2: [(b'f1', f1_1), (b'f2', f1_1)], 3: [(b'f1', f1_1), (b'f2', f1_1)], } c1, c2, c3 = build_commit_graph(self.repo.object_store, commit_spec, trees) self.repo.refs[b"refs/heads/master"] = c3.id self.repo.refs[b"refs/tags/foo"] = c3.id target_path = tempfile.mkdtemp() errstream = BytesIO() self.addCleanup(shutil.rmtree, target_path) r = porcelain.clone(self.repo.path, target_path, checkout=False, errstream=errstream) self.addCleanup(r.close) self.assertEqual(r.path, target_path) target_repo = Repo(target_path) self.assertEqual(0, len(target_repo.open_index())) self.assertEqual(c3.id, target_repo.refs[b'refs/tags/foo']) self.assertTrue(b'f1' not in os.listdir(target_path)) self.assertTrue(b'f2' not in os.listdir(target_path)) c = r.get_config() encoded_path = self.repo.path if not isinstance(encoded_path, bytes): encoded_path = encoded_path.encode('utf-8') self.assertEqual(encoded_path, c.get((b'remote', b'origin'), b'url')) self.assertEqual( b'+refs/heads/*:refs/remotes/origin/*', c.get((b'remote', b'origin'), b'fetch'))
def init(cls, path='.'): """Initialise a new git-papers repo at the given path and commit the basic directory structure.""" # TODO: do we need seperate .db directory? try: Repo(path) except NotGitRepository: pass else: raise RepoInitialised() Repo.init(path) app = cls(path) emptyfiles = ['.index', '.tags', '.toread'] for path in emptyfiles: try: with open(path, 'w'): pass except (IOError, OSError) as e: from shutil import rmtree # TODO: remove created emptyfiles rmtree(osp.join(path, '.git')) raise FileCreationFailed(e.message) app.commit(emptyfiles, INIT) return app
class DulwichAnkiRepo(AnkiRepo): repo_path: Path git: Any = porcelain dulwich_repo: Repo = field(init=False) def __post_init__(self): path_string = str(self.repo_path.resolve()) try: self.dulwich_repo = self.git.init(path_string) except FileExistsError: logger.info(f"Using existing repository at the following path: {self.repo_path}") self.dulwich_repo = Repo(path_string) def stage_all(self): status = self.status() self.dulwich_repo.stage(status.untracked + status.unstaged) def commit(self, message: str = None): if self.there_are_staged_changes(): self.git.commit(self.dulwich_repo, message=message or str(self.status())) def there_are_staged_changes(self): return bool(list(chain(*self.status().staged.values()))) def status(self) -> GitStatus: return self.git.status(self.dulwich_repo)
def get_remote_options(repo_path, prio_remote="origin"): try: repo = Repo(repo_path) except NotGitRepository: # puts("No git repository found!") return None conf = repo.get_config() options = [] for key in conf.keys(): if 'remote' in key: url = conf.get(key, 'url') remote = key[1] option = RemoteOption( url, remote, get_priority( url, remote, prio_remote=prio_remote ) ) options.append(option) options = sorted(options, key=lambda i: i.priority, reverse=True) return options
def TestRepo(): checkout = Repo.init(tempfile.mkdtemp()) with open(os.path.join(checkout.path, 'foo'), 'w') as fp: fp.write('monty') with open(os.path.join(checkout.path, 'bar'), 'w') as fp: fp.write('python') sub_path = os.path.join(checkout.path, 'sub') os.mkdir(sub_path) with open(os.path.join(sub_path, 'foo'), 'w') as fp: fp.write('sub_monty') with open(os.path.join(sub_path, 'bar'), 'w') as fp: fp.write('sub_python') checkout.stage(['foo', 'bar', os.path.join('sub', 'foo'), os.path.join('sub', 'bar')]) checkout.do_commit( 'The first commit', committer='John Doe <*****@*****.**>' ) bare = Repo.init_bare(tempfile.mkdtemp()) client, host_path = get_transport_and_path(checkout.path) refs = client.fetch( host_path, bare, determine_wants=bare.object_store.determine_wants_all, ) bare["HEAD"] = refs["HEAD"] bare["refs/heads/master"] = refs["refs/heads/master"] return bare, checkout
def new(self, path, desc=None, bare=True): """ Create a new bare repo.Local instance. :param path: Path to new repo. :param desc: Repo description. :param bare: Create as bare repo. :returns: New repo.Local instance. """ if os.path.exists(path): raise RepoError('Path already exists: %s' % path) try: os.mkdir(path) if bare: Repo.init_bare(path) else: Repo.init(path) repo = Local(path) if desc: repo.setDescription(desc) version = repo.addVersion() version.save('Box Initialization') return repo except Exception, e: traceback.print_exc() raise RepoError('Error creating repo')
def test_working_tree(self): temp_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, temp_dir) worktree_temp_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, worktree_temp_dir) r = Repo.init(temp_dir) root_sha = r.do_commit( b'empty commit', committer=b'Test Committer <*****@*****.**>', author=b'Test Author <*****@*****.**>', commit_timestamp=12345, commit_timezone=0, author_timestamp=12345, author_timezone=0) r.refs[b'refs/heads/master'] = root_sha w = Repo._init_new_working_directory(worktree_temp_dir, r) new_sha = w.do_commit( b'new commit', committer=b'Test Committer <*****@*****.**>', author=b'Test Author <*****@*****.**>', commit_timestamp=12345, commit_timezone=0, author_timestamp=12345, author_timezone=0) w.refs[b'HEAD'] = new_sha self.assertEqual(os.path.abspath(r.controldir()), os.path.abspath(w.commondir())) self.assertEqual(r.refs.keys(), w.refs.keys()) self.assertNotEqual(r.head(), w.head())
def get(self, repo): service = self.request.get('service') if service: handler_cls = DEFAULT_HANDLERS.get(service) if not handler_cls: self.error(403) return self.response.headers['Content-type'] = 'application/x-%s-advertisement' % service proto = ReceivableProtocol(StringIO().read, self.response.out.write) handler = handler_cls(FileSystemBackend(), [ repo ], proto, stateless_rpc = True, advertise_refs = True) handler.proto.write_pkt_line('# service=%s\n' % service) handler.proto.write_pkt_line(None) handler.handle() else: self.response.headers['Content-type'] = 'text/plain' repo = Repo(repo) refs = repo.get_refs() for name in sorted(refs.iterkeys()): if name == 'HEAD': continue sha = refs[name] if not repo[sha]: continue self.response.out.write('%s\t%s\n' % (sha, name)) peeled_sha = repo.get_peeled(name) if peeled_sha != sha: self.response.out.write('%s\t%s^{}\n' % (peeled_sha, name))
def get_recent_tags(projdir=PROJDIR): """ Get list of recent tags in order from newest to oldest and their datetimes. :param projdir: path to ``.git`` :returns: list of (tag, [datetime, commit, author]) sorted from new to old """ project = Repo(projdir) # dulwich repository object refs = project.get_refs() # dictionary of refs and their SHA-1 values tags = {} # empty dictionary to hold tags, commits and datetimes # iterate over refs in repository for key, value in refs.iteritems(): obj = project.get_object(value) # dulwich object from SHA-1 # check if object is tag if obj.type_name != 'tag': # skip ref if not a tag continue # strip the leading text from "refs/tag/<tag name>" to get "tag name" _, tag = key.rsplit('/', 1) # check if tag object is commit, altho it should always be true if obj.object[0].type_name == 'commit': commit = project.get_object(obj.object[1]) # commit object # get tag commit datetime, but dulwich returns seconds since # beginning of epoch, so use Python time module to convert it to # timetuple then convert to datetime tags[tag] = [ datetime.datetime(*time.gmtime(commit.commit_time)[:6]), commit.id, commit.author ] # return list of tags sorted by their datetimes from newest to oldest return sorted(tags.iteritems(), key=lambda tag: tag[1][0], reverse=True)
def list_all_contributors (self): tmp = 1 tot = len(self.repos) all_contribs = [] for repo in self.repos: print >> sys.stderr, "[%d/%d Analyzing %s]" % (tmp, tot, repo) tmp += 1 repo = Repo(repo) master = repo.get_refs()['refs/heads/master'] for i in repo.get_walker ([master]): if "<" in i.commit.author: split = i.commit.author.split("<") author = split[0] email = split[1] author = author.strip () email = email.strip () email = email[:-1] else: author = i.commit.author email = "" all_contribs.append((author, email)) del repo tmp = [] for c in all_contribs: if c in tmp: continue tmp.append(c) return tmp
class GitBackend(Backend): def __init__(self, gitdir=None): self.gitdir = gitdir if not self.gitdir: self.gitdir = tempfile.mkdtemp() Repo.create(self.gitdir) self.repo = Repo(self.gitdir) self.fetch_objects = self.repo.fetch_objects self.get_refs = self.repo.get_refs def apply_pack(self, refs, read): fd, commit = self.repo.object_store.add_thin_pack() fd.write(read()) fd.close() commit() for oldsha, sha, ref in refs: if ref == "0" * 40: self.repo.remove_ref(ref) else: self.repo.set_ref(ref, sha) print "pack applied"
def test_submodule(self): temp_dir = tempfile.mkdtemp() repo_dir = os.path.join(os.path.dirname(__file__), "data", "repos") shutil.copytree(os.path.join(repo_dir, "a.git"), os.path.join(temp_dir, "a.git"), symlinks=True) rel = os.path.relpath(os.path.join(repo_dir, "submodule"), temp_dir) os.symlink(os.path.join(rel, "dotgit"), os.path.join(temp_dir, ".git")) r = Repo(temp_dir) self.assertEqual(r.head(), "a90fa2d900a17e99b433217e988c4eb4a2e9a097")
def get(self, repo, sha_prefix, sha_suffix): sha = sha_prefix + sha_suffix object_store = Repo(repo).object_store if not object_store.contains_loose(sha): self.error(404) return self.response.headers['Content-type'] = 'application/x-git-loose-object' self.response.out.write(object_store[sha].as_legacy_object())
def _get(self, repo, path, content_type): file = Repo(repo).get_named_file(path) if file: self.response.headers['Content-type'] = content_type self.response.out.write(file.read()) file.close() else: self.error(404)
class ArticleList(object): def __init__(self): self.repo = Repo('wiki') self.head = self.repo.get_object(self.repo.head()) self.tree = self.repo.get_object(self.head.tree) def get_article_titles(self): return [a for a in self.tree]
def create_repository(instance, **kwargs): # Return if the repository has already been created if os.path.exists(instance.path): return # Create the repository and initialize it as a bare Git repo os.makedirs(instance.path) Repo.init_bare(instance.path)
def __init__(self, path=".", extension=".data", autocommit=True): super(GitStore, self).__init__(path, extension) self._autocommit = autocommit gitdir = P.join(self._path, ".git") if P.isdir(gitdir): self._repo = Repo(self._path) else: self._repo = Repo.init(self._path)
def __init__(self, path): try: self.repo = Repo(path) except NotGitRepository: self.repo = Repo.init(path, mkdir=True) # TODO add first commit here self.path = path
def _dulwich_status(self): """ Return the git status """ _repo = Repo(self.config['top_dir']) index = _repo.open_index() return list(tree_changes(_repo, index.commit(_repo.object_store), _repo['HEAD'].tree))
class GitWhoosh: def __init__(self, repos_path, index_path): self.repo = Repo(repos_path) self.index_path = index_path self.git_index = self.repo.open_index() if not exists_in(self.index_path): schema = Schema(path=ID(unique=True, stored=True), itime=STORED, content=TEXT) self.ix = create_in(self.index_path, schema) else: self.ix = open_dir(self.index_path) def hook_index(self, func, path): mtime = self.git_index[path][1] sha = self.git_index[path][8] blob = self.repo.get_blob(sha).as_raw_string() func(path=path.decode("utf-8"), content=blob.decode("utf-8"), itime=mtime) def index(self, regexp=None): with self.ix.searcher() as searcher: writer = self.ix.writer() # first of all, check for removed items paths = {} for fields in searcher.all_stored_fields(): paths[fields["path"]] = fields["itime"] if not fields["path"] in self.git_index: writer.delete_by_term("path", fields["path"]) # now check for new or updated items for path in self.git_index: if regexp: if not re.search(regexp, path): continue if path in paths: if self.git_index[path][1] > paths[path.decode("utf-8")]: self.hook_index(writer.update_document, path) else: self.hook_index(writer.add_document, path) writer.commit() def search(self, query): parser = QueryParser("content", schema=self.ix.schema) q = parser.parse(query.decode("utf-8")) found_items = [] with self.ix.searcher() as searcher: results = searcher.search(q, terms=True) for r in results: terms = [] for term in r.matched_terms(): terms.append(term[1]) found_items.append({"path": r["path"], "terms": terms}) return found_items def __call__(self, environ, start_response): start_response("200 OK", [("Content-Type", "application/json")]) output = [] qs = environ.get("QUERY_STRING", None) if qs: output = self.search(urllib.unquote(qs)) return json.dumps(output)
def _dulwich_commit(self, author, message=DEFAULT_COMMIT_MSG): """ Commit staged files in the repo """ _repo = Repo(self.config['top_dir']) commit_id = _repo.do_commit(message, committer=author) if not _repo.head() == commit_id: raise SartorisError(message=exit_codes[14], exit_code=14)
def get_revision(self): """Read git revision information for the bcfg2 repository""" try: repo = Repo(self.datastore) revision = repo.head() except: logger.error("Failed to read git repository; disabling git support") raise Bcfg2.Server.Plugin.PluginInitError return revision
def _ls_root(self, workspace=None): from dulwich.repo import Repo outstream = StringIO() r = Repo(self.workspace.working_dir) index = r.open_index() for blob in index.iterblobs(): outstream.write('\t'.join(map(str, blob)) + '\n') return ''.join(outstream.getvalue()).encode(), b''
def test_submodule(self): temp_dir = tempfile.mkdtemp() repo_dir = os.path.join(os.path.dirname(__file__), 'data', 'repos') shutil.copytree(os.path.join(repo_dir, 'a.git'), os.path.join(temp_dir, 'a.git'), symlinks=True) rel = os.path.relpath(os.path.join(repo_dir, 'submodule'), temp_dir) os.symlink(os.path.join(rel, 'dotgit'), os.path.join(temp_dir, '.git')) r = Repo(temp_dir) self.assertEqual(r.head(), 'a90fa2d900a17e99b433217e988c4eb4a2e9a097')
def setUp(self): super(FileSystemBackendTests, self).setUp() self.path = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.path) self.repo = Repo.init(self.path) self.backend = FileSystemBackend()
def setUp(self): super(UpdateServerInfoTests, self).setUp() self.path = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.path) self.repo = Repo.init(self.path)
def test_discover_isrepo(self): r = Repo.discover(self._repo_dir) self.assertEqual(r.head(), self._repo.head())
def local_repo(tmpdir_factory: TempdirFactory, source_directory_name: str) -> Repo: with Repo.init(tmpdir_factory.mktemp("src") / source_directory_name, mkdir=True) as repo: yield repo
def test_create_disk_non_bare(self): tmp_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp_dir) repo = Repo.init(tmp_dir) self.assertEqual(os.path.join(tmp_dir, '.git'), repo._controldir) self._check_repo_contents(repo, False)
def test_shell_hook_post_commit(self): if os.name != 'posix': self.skipTest('shell hook tests requires POSIX shell') repo_dir = os.path.join(tempfile.mkdtemp()) r = Repo.init(repo_dir) self.addCleanup(shutil.rmtree, repo_dir) (fd, path) = tempfile.mkstemp(dir=repo_dir) post_commit_msg = """#!/bin/sh rm %(file)s """ % {'file': path} root_sha = r.do_commit('empty commit', committer='Test Committer <*****@*****.**>', author='Test Author <*****@*****.**>', commit_timestamp=12345, commit_timezone=0, author_timestamp=12345, author_timezone=0) self.assertEqual([], r[root_sha].parents) post_commit = os.path.join(r.controldir(), 'hooks', 'post-commit') with open(post_commit, 'wb') as f: f.write(post_commit_msg) os.chmod(post_commit, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) commit_sha = r.do_commit( 'empty commit', committer='Test Committer <*****@*****.**>', author='Test Author <*****@*****.**>', commit_timestamp=12345, commit_timezone=0, author_timestamp=12345, author_timezone=0) self.assertEqual([root_sha], r[commit_sha].parents) self.assertFalse(os.path.exists(path)) post_commit_msg_fail = """#!/bin/sh exit 1 """ with open(post_commit, 'wb') as f: f.write(post_commit_msg_fail) os.chmod(post_commit, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) warnings.simplefilter("always", UserWarning) self.addCleanup(warnings.resetwarnings) warnings_list, restore_warnings = setup_warning_catcher() self.addCleanup(restore_warnings) commit_sha2 = r.do_commit( 'empty commit', committer='Test Committer <*****@*****.**>', author='Test Author <*****@*****.**>', commit_timestamp=12345, commit_timezone=0, author_timestamp=12345, author_timezone=0) self.assertEqual(len(warnings_list), 1) self.assertIsInstance(warnings_list[-1], UserWarning) self.assertTrue("post-commit hook failed: " in str(warnings_list[-1])) self.assertEqual([commit_sha], r[commit_sha2].parents)
#!/usr/bin/env python3 from dulwich.repo import Repo r = Repo('.') h = r.head() print(h) c = r[h].message print(c)
class DiskRefsContainerTests(RefsContainerTests, TestCase): def setUp(self): TestCase.setUp(self) self._repo = open_repo('refs.git') self.addCleanup(tear_down_repo, self._repo) self._refs = self._repo.refs def test_get_packed_refs(self): self.assertEqual( { b'refs/heads/packed': b'42d06bd4b77fed026b154d16493e5deab78f02ec', b'refs/tags/refs-0.1': b'df6800012397fb85c56e7418dd4eb9405dee075c', }, self._refs.get_packed_refs()) def test_get_peeled_not_packed(self): # not packed self.assertEqual(None, self._refs.get_peeled(b'refs/tags/refs-0.2')) self.assertEqual(b'3ec9c43c84ff242e3ef4a9fc5bc111fd780a76a8', self._refs[b'refs/tags/refs-0.2']) # packed, known not peelable self.assertEqual(self._refs[b'refs/heads/packed'], self._refs.get_peeled(b'refs/heads/packed')) # packed, peeled self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec', self._refs.get_peeled(b'refs/tags/refs-0.1')) def test_setitem(self): RefsContainerTests.test_setitem(self) f = open(os.path.join(self._refs.path, 'refs', 'some', 'ref'), 'rb') self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec', f.read()[:40]) f.close() def test_setitem_symbolic(self): ones = b'1' * 40 self._refs[b'HEAD'] = ones self.assertEqual(ones, self._refs[b'HEAD']) # ensure HEAD was not modified f = open(os.path.join(self._refs.path, 'HEAD'), 'rb') self.assertEqual(b'ref: refs/heads/master', next(iter(f)).rstrip(b'\n')) f.close() # ensure the symbolic link was written through f = open(os.path.join(self._refs.path, 'refs', 'heads', 'master'), 'rb') self.assertEqual(ones, f.read()[:40]) f.close() def test_set_if_equals(self): RefsContainerTests.test_set_if_equals(self) # ensure symref was followed self.assertEqual(b'9' * 40, self._refs[b'refs/heads/master']) # ensure lockfile was deleted self.assertFalse( os.path.exists( os.path.join(self._refs.path, 'refs', 'heads', 'master.lock'))) self.assertFalse( os.path.exists(os.path.join(self._refs.path, 'HEAD.lock'))) def test_add_if_new_packed(self): # don't overwrite packed ref self.assertFalse( self._refs.add_if_new(b'refs/tags/refs-0.1', b'9' * 40)) self.assertEqual(b'df6800012397fb85c56e7418dd4eb9405dee075c', self._refs[b'refs/tags/refs-0.1']) def test_add_if_new_symbolic(self): # Use an empty repo instead of the default. repo_dir = os.path.join(tempfile.mkdtemp(), 'test') os.makedirs(repo_dir) repo = Repo.init(repo_dir) self.addCleanup(tear_down_repo, repo) refs = repo.refs nines = b'9' * 40 self.assertEqual(b'ref: refs/heads/master', refs.read_ref(b'HEAD')) self.assertFalse(b'refs/heads/master' in refs) self.assertTrue(refs.add_if_new(b'HEAD', nines)) self.assertEqual(b'ref: refs/heads/master', refs.read_ref(b'HEAD')) self.assertEqual(nines, refs[b'HEAD']) self.assertEqual(nines, refs[b'refs/heads/master']) self.assertFalse(refs.add_if_new(b'HEAD', b'1' * 40)) self.assertEqual(nines, refs[b'HEAD']) self.assertEqual(nines, refs[b'refs/heads/master']) def test_follow(self): self.assertEqual(([b'HEAD', b'refs/heads/master' ], b'42d06bd4b77fed026b154d16493e5deab78f02ec'), self._refs.follow(b'HEAD')) self.assertEqual(([b'refs/heads/master' ], b'42d06bd4b77fed026b154d16493e5deab78f02ec'), self._refs.follow(b'refs/heads/master')) self.assertRaises(KeyError, self._refs.follow, b'refs/heads/loop') def test_delitem(self): RefsContainerTests.test_delitem(self) ref_file = os.path.join(self._refs.path, 'refs', 'heads', 'master') self.assertFalse(os.path.exists(ref_file)) self.assertFalse(b'refs/heads/master' in self._refs.get_packed_refs()) def test_delitem_symbolic(self): self.assertEqual(b'ref: refs/heads/master', self._refs.read_loose_ref(b'HEAD')) del self._refs[b'HEAD'] self.assertRaises(KeyError, lambda: self._refs[b'HEAD']) self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec', self._refs[b'refs/heads/master']) self.assertFalse(os.path.exists(os.path.join(self._refs.path, 'HEAD'))) def test_remove_if_equals_symref(self): # HEAD is a symref, so shouldn't equal its dereferenced value self.assertFalse( self._refs.remove_if_equals( b'HEAD', b'42d06bd4b77fed026b154d16493e5deab78f02ec')) self.assertTrue( self._refs.remove_if_equals( b'refs/heads/master', b'42d06bd4b77fed026b154d16493e5deab78f02ec')) self.assertRaises(KeyError, lambda: self._refs[b'refs/heads/master']) # HEAD is now a broken symref self.assertRaises(KeyError, lambda: self._refs[b'HEAD']) self.assertEqual(b'ref: refs/heads/master', self._refs.read_loose_ref(b'HEAD')) self.assertFalse( os.path.exists( os.path.join(self._refs.path, 'refs', 'heads', 'master.lock'))) self.assertFalse( os.path.exists(os.path.join(self._refs.path, 'HEAD.lock'))) def test_remove_packed_without_peeled(self): refs_file = os.path.join(self._repo.path, 'packed-refs') f = GitFile(refs_file) refs_data = f.read() f.close() f = GitFile(refs_file, 'wb') f.write(b'\n'.join(l for l in refs_data.split(b'\n') if not l or l[0] not in b'#^')) f.close() self._repo = Repo(self._repo.path) refs = self._repo.refs self.assertTrue( refs.remove_if_equals(b'refs/heads/packed', b'42d06bd4b77fed026b154d16493e5deab78f02ec')) def test_remove_if_equals_packed(self): # test removing ref that is only packed self.assertEqual(b'df6800012397fb85c56e7418dd4eb9405dee075c', self._refs[b'refs/tags/refs-0.1']) self.assertTrue( self._refs.remove_if_equals( b'refs/tags/refs-0.1', b'df6800012397fb85c56e7418dd4eb9405dee075c')) self.assertRaises(KeyError, lambda: self._refs[b'refs/tags/refs-0.1']) def test_read_ref(self): self.assertEqual(b'ref: refs/heads/master', self._refs.read_ref(b'HEAD')) self.assertEqual(b'42d06bd4b77fed026b154d16493e5deab78f02ec', self._refs.read_ref(b'refs/heads/packed')) self.assertEqual(None, self._refs.read_ref(b'nonexistant')) def test_non_ascii(self): try: encoded_ref = u'refs/tags/schön'.encode( sys.getfilesystemencoding()) except UnicodeEncodeError: raise SkipTest( "filesystem encoding doesn't support special character") p = os.path.join(self._repo.path, 'refs', 'tags', 'schön') with open(p, 'w') as f: f.write('00' * 20) expected_refs = dict(_TEST_REFS) expected_refs[encoded_ref] = b'00' * 20 self.assertEqual(expected_refs, self._repo.get_refs())
"""Read the config file for a git repository. Example usage: python examples/config.py """ from dulwich.repo import Repo repo = Repo(".") config = repo.get_config() print(config.get("core", "filemode")) print(config.get(("remote", "origin"), "url"))
def clone(source, target=None, bare=False, checkout=None, errstream=default_bytes_err_stream, outstream=None, origin=b"origin"): """Clone a local or remote git repository. :param source: Path or URL for source repository :param target: Path to target repository (optional) :param bare: Whether or not to create a bare repository :param checkout: Whether or not to check-out HEAD after cloning :param errstream: Optional stream to write progress to :param outstream: Optional stream to write progress to (deprecated) :return: The new repository """ if outstream is not None: import warnings warnings.warn( "outstream= has been deprecated in favour of errstream=.", DeprecationWarning, stacklevel=3) errstream = outstream if checkout is None: checkout = (not bare) if checkout and bare: raise ValueError("checkout and bare are incompatible") client, host_path = get_transport_and_path(source) if target is None: target = host_path.split("/")[-1] if not os.path.exists(target): os.mkdir(target) if bare: r = Repo.init_bare(target) else: r = Repo.init(target) try: remote_refs = client.fetch( host_path, r, determine_wants=r.object_store.determine_wants_all, progress=errstream.write) r.refs.import_refs( b'refs/remotes/' + origin, { n[len(b'refs/heads/'):]: v for (n, v) in remote_refs.items() if n.startswith(b'refs/heads/') }) r.refs.import_refs( b'refs/tags', { n[len(b'refs/tags/'):]: v for (n, v) in remote_refs.items() if n.startswith(b'refs/tags/') and not n.endswith(ANNOTATED_TAG_SUFFIX) }) r[b"HEAD"] = remote_refs[b"HEAD"] target_config = r.get_config() if not isinstance(source, bytes): source = source.encode(DEFAULT_ENCODING) target_config.set((b'remote', b'origin'), b'url', source) target_config.set((b'remote', b'origin'), b'fetch', b'+refs/heads/*:refs/remotes/origin/*') target_config.write_to_path() if checkout: errstream.write(b'Checking out HEAD\n') r.reset_index() except: r.close() raise return r
def test_open_existing(self): r = GitRepo.init('.') d = ControlDir.open('.') thebranch = d.create_branch() self.assertIsInstance(thebranch, branch.GitBranch)
class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method """Dulwich Git backend.""" from dulwich import client from .asyncssh_vendor import AsyncSSHVendor # monkeypatch dulwich client's default SSH vendor to use asyncssh client.get_ssh_vendor = AsyncSSHVendor # type: ignore[assignment] # Dulwich progress will return messages equivalent to git CLI, # our pbars should just display the messages as formatted by dulwich BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]} [{elapsed}]" def __init__( # pylint:disable=W0231 self, root_dir=os.curdir, search_parent_directories=True): from dulwich.errors import NotGitRepository from dulwich.repo import Repo try: if search_parent_directories: self.repo = Repo.discover(start=root_dir) else: self.repo = Repo(root_dir) except NotGitRepository as exc: raise SCMError(f"{root_dir} is not a git repository") from exc self._submodules: Dict[str, "PathInfo"] = self._find_submodules() self._stashes: dict = {} def _find_submodules(self) -> Dict[str, "PathInfo"]: """Return dict mapping submodule names to submodule paths. Submodule paths will be relative to Git repo root. """ from dulwich.config import ConfigFile, parse_submodules submodules: Dict[str, "PathInfo"] = {} config_path = os.path.join(self.root_dir, ".gitmodules") if os.path.isfile(config_path): config = ConfigFile.from_path(config_path) for path, _url, section in parse_submodules(config): submodules[os.fsdecode(section)] = PathInfo(os.fsdecode(path)) return submodules def close(self): self.repo.close() @property def root_dir(self) -> str: return self.repo.path @staticmethod def clone( url: str, to_path: str, rev: Optional[str] = None, shallow_branch: Optional[str] = None, ): raise NotImplementedError @property def dir(self) -> str: return self.repo.commondir() def add(self, paths: Union[str, Iterable[str]], update=False): from dvc.utils.fs import walk_files assert paths or update if isinstance(paths, str): paths = [paths] if update and not paths: self.repo.stage(list(self.repo.open_index())) return files: List[bytes] = [] for path in paths: if not os.path.isabs(path) and self._submodules: # NOTE: If path is inside a submodule, Dulwich expects the # staged paths to be relative to the submodule root (not the # parent git repo root). We append path to root_dir here so # that the result of relpath(path, root_dir) is actually the # path relative to the submodule root. path_info = PathInfo(path).relative_to(self.root_dir) for sm_path in self._submodules.values(): if path_info.isin(sm_path): path = os.path.join(self.root_dir, path_info.relative_to(sm_path)) break if os.path.isdir(path): files.extend( os.fsencode(relpath(fpath, self.root_dir)) for fpath in walk_files(path)) else: files.append(os.fsencode(relpath(path, self.root_dir))) # NOTE: this doesn't check gitignore, same as GitPythonBackend.add if update: index = self.repo.open_index() if os.name == "nt": # NOTE: we need git/unix separator to compare against index # paths but repo.stage() expects to be called with OS paths self.repo.stage([ fname for fname in files if fname.replace(b"\\", b"/") in index ]) else: self.repo.stage([fname for fname in files if fname in index]) else: self.repo.stage(files) def commit(self, msg: str, no_verify: bool = False): from dulwich.errors import CommitError from dulwich.porcelain import commit from dulwich.repo import InvalidUserIdentity try: commit(self.root_dir, message=msg, no_verify=no_verify) except CommitError as exc: raise SCMError("Git commit failed") from exc except InvalidUserIdentity as exc: raise SCMError( "Git username and email must be configured") from exc def checkout( self, branch: str, create_new: Optional[bool] = False, force: bool = False, **kwargs, ): raise NotImplementedError def pull(self, **kwargs): raise NotImplementedError def push(self): raise NotImplementedError def branch(self, branch: str): from dulwich.porcelain import Error, branch_create try: branch_create(self.root_dir, branch) except Error as exc: raise SCMError(f"Failed to create branch '{branch}'") from exc def tag(self, tag: str): raise NotImplementedError def untracked_files(self) -> Iterable[str]: _staged, _unstaged, untracked = self.status() return untracked def is_tracked(self, path: str) -> bool: rel = PathInfo(path).relative_to(self.root_dir).as_posix().encode() rel_dir = rel + b"/" for path in self.repo.open_index(): if path == rel or path.startswith(rel_dir): return True return False def is_dirty(self, untracked_files: bool = False) -> bool: staged, unstaged, untracked = self.status() return bool(staged or unstaged or (untracked_files and untracked)) def active_branch(self) -> str: raise NotImplementedError def list_branches(self) -> Iterable[str]: raise NotImplementedError def list_tags(self) -> Iterable[str]: raise NotImplementedError def list_all_commits(self) -> Iterable[str]: raise NotImplementedError def get_tree_obj(self, rev: str, **kwargs) -> DulwichObject: from dulwich.objectspec import parse_tree tree = parse_tree(self.repo, rev) return DulwichObject(self.repo, ".", stat.S_IFDIR, tree.id) def get_rev(self) -> str: rev = self.get_ref("HEAD") if rev: return rev raise SCMError("Empty git repo") def resolve_rev(self, rev: str) -> str: raise NotImplementedError def resolve_commit(self, rev: str) -> "GitCommit": raise NotImplementedError def _get_stash(self, ref: str): from dulwich.stash import Stash as DulwichStash if ref not in self._stashes: self._stashes[ref] = DulwichStash(self.repo, ref=os.fsencode(ref)) return self._stashes[ref] @cached_property def ignore_manager(self): from dulwich.ignore import IgnoreFilterManager return IgnoreFilterManager.from_repo(self.repo) def is_ignored(self, path: "StrPath") -> bool: # `is_ignored` returns `false` if excluded in `.gitignore` and # `None` if it's not mentioned at all. `True` if it is ignored. relative_path = relpath(path, self.root_dir) # if checking a directory, a trailing slash must be included if str(path)[-1] == os.sep: relative_path += os.sep return bool(self.ignore_manager.is_ignored(relative_path)) def set_ref( self, name: str, new_ref: str, old_ref: Optional[str] = None, message: Optional[str] = None, symbolic: Optional[bool] = False, ): name_b = os.fsencode(name) new_ref_b = os.fsencode(new_ref) old_ref_b = os.fsencode(old_ref) if old_ref else None message_b = message.encode("utf-8") if message else None if symbolic: return self.repo.refs.set_symbolic_ref(name_b, new_ref_b, message=message_b) if not self.repo.refs.set_if_equals( name_b, old_ref_b, new_ref_b, message=message_b): raise SCMError(f"Failed to set '{name}'") def get_ref(self, name, follow: bool = True) -> Optional[str]: from dulwich.refs import parse_symref_value name_b = os.fsencode(name) if follow: try: ref = self.repo.refs[name_b] except KeyError: ref = None else: ref = self.repo.refs.read_ref(name_b) try: if ref: ref = parse_symref_value(ref) except ValueError: pass if ref: return os.fsdecode(ref) return None def remove_ref(self, name: str, old_ref: Optional[str] = None): name_b = name.encode("utf-8") old_ref_b = old_ref.encode("utf-8") if old_ref else None if not self.repo.refs.remove_if_equals(name_b, old_ref_b): raise SCMError(f"Failed to remove '{name}'") def iter_refs(self, base: Optional[str] = None): base_b = os.fsencode(base) if base else None for key in self.repo.refs.keys(base=base_b): if base: if base.endswith("/"): base = base[:-1] yield "/".join([base, os.fsdecode(key)]) else: yield os.fsdecode(key) def iter_remote_refs(self, url: str, base: Optional[str] = None, **kwargs): from dulwich.client import HTTPUnauthorized, get_transport_and_path from dulwich.errors import NotGitRepository from dulwich.porcelain import get_remote_repo try: _remote, location = get_remote_repo(self.repo, url) client, path = get_transport_and_path(location, **kwargs) except Exception as exc: raise InvalidRemoteSCMRepo(url) from exc try: if base: yield from (os.fsdecode(ref) for ref in client.get_refs(path) if ref.startswith(os.fsencode(base))) else: yield from (os.fsdecode(ref) for ref in client.get_refs(path)) except NotGitRepository as exc: raise InvalidRemoteSCMRepo(url) from exc except HTTPUnauthorized: raise GitAuthError(url) def get_refs_containing(self, rev: str, pattern: Optional[str] = None): raise NotImplementedError def push_refspec( self, url: str, src: Optional[str], dest: str, force: bool = False, on_diverged: Optional[Callable[[str, str], bool]] = None, **kwargs, ): from dulwich.client import HTTPUnauthorized, get_transport_and_path from dulwich.errors import NotGitRepository, SendPackError from dulwich.porcelain import ( DivergedBranches, check_diverged, get_remote_repo, ) dest_refs, values = self._push_dest_refs(src, dest) try: _remote, location = get_remote_repo(self.repo, url) client, path = get_transport_and_path(location, **kwargs) except Exception as exc: raise SCMError( f"'{url}' is not a valid Git remote or URL") from exc def update_refs(refs): from dulwich.objects import ZERO_SHA new_refs = {} for ref, value in zip(dest_refs, values): if ref in refs and value != ZERO_SHA: local_sha = self.repo.refs[ref] remote_sha = refs[ref] try: check_diverged(self.repo, remote_sha, local_sha) except DivergedBranches: if not force: overwrite = False if on_diverged: overwrite = on_diverged( os.fsdecode(ref), os.fsdecode(remote_sha)) if not overwrite: continue new_refs[ref] = value return new_refs try: with Tqdm(desc="Pushing git refs", bar_format=self.BAR_FMT_NOTOTAL) as pbar: def progress(msg_b): msg = msg_b.decode("ascii").strip() pbar.update_msg(msg) pbar.refresh() logger.trace(msg) client.send_pack( path, update_refs, self.repo.object_store.generate_pack_data, progress=progress, ) except (NotGitRepository, SendPackError) as exc: raise SCMError("Git failed to push '{src}' to '{url}'") from exc except HTTPUnauthorized: raise GitAuthError(url) def _push_dest_refs(self, src: Optional[str], dest: str) -> Tuple[Iterable[bytes], Iterable[bytes]]: from dulwich.objects import ZERO_SHA if src is not None and src.endswith("/"): src_b = os.fsencode(src) keys = self.repo.refs.subkeys(src_b) values = [self.repo.refs[b"".join([src_b, key])] for key in keys] dest_refs = [b"".join([os.fsencode(dest), key]) for key in keys] else: if src is None: values = [ZERO_SHA] else: values = [self.repo.refs[os.fsencode(src)]] dest_refs = [os.fsencode(dest)] return dest_refs, values def fetch_refspecs( self, url: str, refspecs: Iterable[str], force: Optional[bool] = False, on_diverged: Optional[Callable[[str, str], bool]] = None, **kwargs, ): from dulwich.client import get_transport_and_path from dulwich.objectspec import parse_reftuples from dulwich.porcelain import ( DivergedBranches, check_diverged, get_remote_repo, ) fetch_refs = [] def determine_wants(remote_refs): fetch_refs.extend( parse_reftuples( remote_refs, self.repo.refs, [os.fsencode(refspec) for refspec in refspecs], force=force, )) return [ remote_refs[lh] for (lh, _, _) in fetch_refs if remote_refs[lh] not in self.repo.object_store ] try: _remote, location = get_remote_repo(self.repo, url) client, path = get_transport_and_path(location, **kwargs) except Exception as exc: raise SCMError( f"'{url}' is not a valid Git remote or URL") from exc with Tqdm(desc="Fetching git refs", bar_format=self.BAR_FMT_NOTOTAL) as pbar: def progress(msg_b): msg = msg_b.decode("ascii").strip() pbar.update_msg(msg) pbar.refresh() logger.trace(msg) fetch_result = client.fetch( path, self.repo, progress=progress, determine_wants=determine_wants, ) for (lh, rh, _) in fetch_refs: try: if rh in self.repo.refs: check_diverged(self.repo, self.repo.refs[rh], fetch_result.refs[lh]) except DivergedBranches: if not force: overwrite = False if on_diverged: overwrite = on_diverged( os.fsdecode(rh), os.fsdecode(fetch_result.refs[lh])) if not overwrite: continue self.repo.refs[rh] = fetch_result.refs[lh] def _stash_iter(self, ref: str): stash = self._get_stash(ref) yield from stash.stashes() def _stash_push( self, ref: str, message: Optional[str] = None, include_untracked: Optional[bool] = False, ) -> Tuple[Optional[str], bool]: from dulwich.repo import InvalidUserIdentity from dvc.scm.git import Stash if include_untracked or ref == Stash.DEFAULT_STASH: # dulwich stash.push does not support include_untracked and does # not touch working tree raise NotImplementedError stash = self._get_stash(ref) message_b = message.encode("utf-8") if message else None try: rev = stash.push(message=message_b) except InvalidUserIdentity as exc: raise SCMError( "Git username and email must be configured") from exc return os.fsdecode(rev), True def _stash_apply(self, rev: str): raise NotImplementedError def _stash_drop(self, ref: str, index: int): from dvc.scm.git import Stash if ref == Stash.DEFAULT_STASH: raise NotImplementedError stash = self._get_stash(ref) try: stash.drop(index) except ValueError as exc: raise SCMError("Failed to drop stash entry") from exc def describe( self, rev: str, base: Optional[str] = None, match: Optional[str] = None, exclude: Optional[str] = None, ) -> Optional[str]: if not base: base = "refs/tags" for ref in self.iter_refs(base=base): if (match and not fnmatch.fnmatch(ref, match)) or ( exclude and fnmatch.fnmatch(ref, exclude)): continue if self.get_ref(ref, follow=False) == rev: return ref return None def diff(self, rev_a: str, rev_b: str, binary=False) -> str: from dulwich.patch import write_tree_diff commit_a = self.repo[os.fsencode(rev_a)] commit_b = self.repo[os.fsencode(rev_b)] buf = BytesIO() write_tree_diff(buf, self.repo.object_store, commit_a.tree, commit_b.tree) return buf.getvalue().decode("utf-8") def reset(self, hard: bool = False, paths: Iterable[str] = None): raise NotImplementedError def checkout_index( self, paths: Optional[Iterable[str]] = None, force: bool = False, ours: bool = False, theirs: bool = False, ): raise NotImplementedError def status( self, ignored: bool = False ) -> Tuple[Mapping[str, Iterable[str]], Iterable[str], Iterable[str]]: from dulwich.porcelain import status as git_status staged, unstaged, untracked = git_status(self.root_dir, ignored=ignored) return ( { status: [os.fsdecode(name) for name in paths] for status, paths in staged.items() if paths }, [os.fsdecode(name) for name in unstaged], [os.fsdecode(name) for name in untracked], ) def _reset(self) -> None: self.__dict__.pop("ignore_manager", None) def merge( self, rev: str, commit: bool = True, msg: Optional[str] = None, squash: bool = False, ) -> Optional[str]: raise NotImplementedError def validate_git_remote(self, url: str, **kwargs): from dulwich.client import LocalGitClient, get_transport_and_path from dulwich.porcelain import get_remote_repo try: _, location = get_remote_repo(self.repo, url) client, path = get_transport_and_path(location, **kwargs) except Exception as exc: raise InvalidRemoteSCMRepo(url) from exc if isinstance(client, LocalGitClient) and not os.path.exists( os.path.join("", path)): raise InvalidRemoteSCMRepo(url) def check_ref_format(self, refname: str): from dulwich.refs import check_ref_format return check_ref_format(refname.encode())
def test_discover_intended(self): path = os.path.join(self._repo_dir, 'b/c') r = Repo.discover(path) self.assertEqual(r.head(), self._repo.head())
def repository(self): return Repo(self.local_path)
def open_repository(self, path): logger.debug('opening repository at %s', path) return Repo(path)
def test_discover_notrepo(self): with self.assertRaises(NotGitRepository): Repo.discover('/')
def test_create_disk_bare(self): tmp_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp_dir) repo = Repo.init_bare(tmp_dir) self.assertEqual(tmp_dir, repo._controldir) self._check_repo_contents(repo, True)
def load_repo(self, repo_name): self.repo_name = repo_name self.repo = DulwichRepo(self.repo_name) self.current_dir = os.getcwd() self.repo_path = self.current_dir + '/' + self.repo_name
def __init__(self, path): self.repo = DulwichRepo(path) # The inner Dulwich Repo object. self.root = path
def test_shell_hook_post_commit(self): if os.name != 'posix': self.skipTest('shell hook tests requires POSIX shell') repo_dir = self.mkdtemp() self.addCleanup(shutil.rmtree, repo_dir) r = Repo.init(repo_dir) self.addCleanup(r.close) (fd, path) = tempfile.mkstemp(dir=repo_dir) os.close(fd) post_commit_msg = """#!/bin/sh rm """ + path + """ """ root_sha = r.do_commit(b'empty commit', committer=b'Test Committer <*****@*****.**>', author=b'Test Author <*****@*****.**>', commit_timestamp=12345, commit_timezone=0, author_timestamp=12345, author_timezone=0) self.assertEqual([], r[root_sha].parents) post_commit = os.path.join(r.controldir(), 'hooks', 'post-commit') with open(post_commit, 'wb') as f: f.write(post_commit_msg.encode(locale.getpreferredencoding())) os.chmod(post_commit, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) commit_sha = r.do_commit( b'empty commit', committer=b'Test Committer <*****@*****.**>', author=b'Test Author <*****@*****.**>', commit_timestamp=12345, commit_timezone=0, author_timestamp=12345, author_timezone=0) self.assertEqual([root_sha], r[commit_sha].parents) self.assertFalse(os.path.exists(path)) post_commit_msg_fail = """#!/bin/sh exit 1 """ with open(post_commit, 'w') as f: f.write(post_commit_msg_fail) os.chmod(post_commit, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) warnings.simplefilter("always", UserWarning) self.addCleanup(warnings.resetwarnings) warnings_list, restore_warnings = setup_warning_catcher() self.addCleanup(restore_warnings) commit_sha2 = r.do_commit( b'empty commit', committer=b'Test Committer <*****@*****.**>', author=b'Test Author <*****@*****.**>', commit_timestamp=12345, commit_timezone=0, author_timestamp=12345, author_timezone=0) expected_warning = UserWarning( 'post-commit hook failed: Hook post-commit exited with ' 'non-zero status 1', ) for w in warnings_list: if (type(w) == type(expected_warning) and w.args == expected_warning.args): break else: raise AssertionError('Expected warning %r not in %r' % (expected_warning, warnings_list)) self.assertEqual([commit_sha], r[commit_sha2].parents)
class backend(): def __init__(self): self.username = "" self.email = "" self.activity = "" self.repo_path = "" self.repo_name = "" self.isaclone = 0 self.cloned_from = "" def set_authorinfo(self, username, email): self.username = username self.email = email def local_init(self, repo_name, activity): self.activity = activity self.repo_name = repo_name try: self.repo = p.init(repo_name) self.current_dir = os.getcwd() self.repo_path = self.current_dir + '/' + self.repo_name print self.repo_path print "Local Repo Created" except: print "Repo already exist, delete it first" def load_repo(self, repo_name): self.repo_name = repo_name self.repo = DulwichRepo(self.repo_name) self.current_dir = os.getcwd() self.repo_path = self.current_dir + '/' + self.repo_name def create_file(self, name, content): try: file = open(os.path.join(self.repo_path, name), 'w') file.write(content) file.close() except: print 'Unable to create README, does it already exist?' def edit_readme(self, name, content): file = open(os.path.join(self.repo_path, name), 'w') file.write(content) file.close() def add(self, a): #a can be list of files or a single file print self.repo_name print self.repo if type(a) == list: for i in a: p.add(self.repo, i) else: p.add(self.repo, a) def get_status(self): if os.path.exists(self.repo_path): print self.repo_path print p.status(self.repo_path) else: print "Repo does not exist" def commit(self, message): p.commit(self.repo, message) def get_commit_history(self): print self.repo_path r = self.repo f = "README" w = r.get_walker(paths=[f], max_entries=None) count = 0 for i in iter(w): count += 1 print count, print i print i.commit def clone_local(self, clone_repo_name): #Creating a clone of a given repo. The repo should be local. p.clone(self.repo_path, clone_repo_name) def clone_remote(remote_repo_name, clone_repo_name): #Creating a clone of remote repo. p.clone(remote_repo_name, clone_repo_name) def commit_logs(self): try: if os.path.exists(self.repo_path): print p.log(self.repo) else: print "Repo does not exist" except: print "No commits yet" """ #Some issues - have to be rectified asap def revert_to_commit(self): print self.repo_path r = self.repo f = "README" w = r.get_walker(paths=[], max_entries=None) count = 0 for i in iter(w): count += 1 print count, print type(i) print i print i.commit.id a = i.commit.id #a = a[0:8] #print i.commit.get_sha_for() print a p.reset(self.repo, "hard", a) """ def get_diff(self): #p.diff_tree(self.repo,) f = "README" tree_list = [] w = self.repo.get_walker(paths=[f], max_entries=None) for i in iter(w): tree_list.append(i.commit.tree) print i.commit.tree print len(tree_list) p.diff_tree(self.repo, tree_list[0], tree_list[3]) def update_local(self): if self.isaclone == 1: try: p.pull(self.repo, self.cloned_from) except: print "Error" else: print "Can not update"
def test_init_with_empty_info_grafts(self): r = self._repo r._put_named_file(os.path.join('info', 'grafts'), '') r = Repo(self._repo_dir) self.assertEqual({}, r._graftpoints)
class Repo(object): """ An abstraction layer on top of dulwich.repo.Repo for higher-level git repository actions like: * adding only modified files * checking out whole trees (or paths within them) from refs * diffs with difflib * branching and tagging (both displaying and creating) * listing commits down from a ref Methods are structured to match git commands when appropriate. It also supports executing arbitrary git commands, if git is installed. Of course, everything else is implemented in pure python, so having git installed is optional. Should be considered a work-in-progress. """ def __init__(self, path): self.repo = DulwichRepo(path) # The inner Dulwich Repo object. self.root = path @classmethod def init(cls, path, mkdir=False, bare=False): """ Initializes a normal or bare repository. This is mostly a handoff to Dulwich. :param path: the path (which must be a directory) to create the repository within. :param mkdir: if True, make a directory at **path**. Equivalent to ``mkdir [path] && cd [path] && git init``. :param bare: if True, create a bare repository at the path. :return: a ``Repo`` instance. """ if bare: DulwichRepo.init_bare(path) else: DulwichRepo.init(path, mkdir) return cls(path) def add(self, path=None, all=False, add_new_files=True): """ Add files to the repository or staging area if new or modified. Equivalent to the ``git add`` command. :param path: the path to the file to add, relative to the repository root. :param all: if True, add all files under the given path. If **path** is omitted, the repository's root path will be used. :param add_new_files: if True, this command will also add new files. Note this is the default behavior. The option is provided for situations (e.g. ``git commit -a``) where adding new files would be undesirable. :return: list of filepaths that were added. If **path** is a file and **all** is True, only the single file will be added. If **path** is a directory and **all** is False, nothing will be added. Likewise, if both **path** and **all** are omitted, nothing will be added. Additionally, the ``add`` method checks to see if the path(s) have been modified. We don't want to create new blobs if we don't need them. """ # the implementation creates a list of paths and stages them using # dulwich.Repo.stage # Paths are a little tricky. To work with repositories independent # of the current working directory, we need absolute paths to files. # At the same time, git trees are relative to the repository root. # So, we have to do a few conversions. adds = [] # get an absolute path for doing isfile/isdir checks. if path is not None: path = os.path.join(self.root, path) # add all files within given path if path is not None and all: if os.path.isdir(path): # walk the directory for directory, dirnames, filenames in os.walk(directory): if '.git' in dirnames: # in case path is root, don't traverse the .git subdir dirnames.remove('.git') for f in filenames: path = os.path.join(directory, f) adds.append(path) elif os.path.isfile(path): adds.append(path) # add all files within root path elif path is None and all: # walk the root directory for directory, dirnames, filenames in os.walk(self.root): if '.git' in dirnames: # don't traverse the .git subdir dirnames.remove('.git') for f in filenames: path = os.path.join(directory, f) adds.append(path) # add file at path elif path is not None: # add only if file if os.path.isfile(path): adds.append(path) # back to relative paths, so we can add them to the tree. rels = [] for p in adds: # get the path relative to repo root. rels.append(os.path.relpath(p, self.root)) adds = rels # filter unmodified files (and untracked files if not add_new_files) if add_new_files: adds = [f for f in adds if self._file_is_modified(f) or \ not self._file_in_tree(f)] else: adds = [f for f in adds if self._file_is_modified(f)] # don't waste time with stage if empty list. if adds: self.repo.stage(adds) return adds def branch(self, name=None, ref=None): """ Create a new branch or display the current one. Equivalent to `git branch`. :param name: the name of the branch :param ref: a commit reference (branch, tag, or SHA). Same idea as the git-branch ``--start-point`` option. Will create the branch off of the commit. Defaults to HEAD. :return: None on create, branch name on display. When the name param is not given, the current branch will be returned as a string using the branch's full name (i.e. ``refs/heads/[branch_name]``). """ # create a branch if name is not None: if ref is None: ref = self.head().id else: ref = self._resolve_ref(ref) self.repo.refs['refs/heads/%s' % name] = ref # display the name of the current branch else: # couldn't find an easy way to get it out of dulwich, # which resolves HEAD to the commit, so we'll just read # .git/HEAD directly. path = os.path.join(self.repo._controldir, 'HEAD') if os.path.isfile(path): with open(path, 'r') as fp: return fp.read().strip()[5:] def checkout(self, ref, path=None): """ Checkout the entire tree (or a subset) of a commit given a branch, tag, or commit SHA. This is a fairly naive implementation. It will just write the blob data recursively from the tree pointed at by the given reference, overwriting the working tree as necessary. It doesn't do deletions or renames. If you wanted to checkout 'HEAD': >>> repo.checkout(repo.head()) If you wanted to checkout the master branch: >>> repo.checkout('master') If you wanted to checkout v1.2 (i.e. a tag): >>> repo.checkout('v1.2') :param ref: branch, tag, or commit :param path: checkout only file or directory at path, should be relative to the repo's root. :raises KeyError: if bad reference. """ sha = self._resolve_ref(ref) obj = self.repo[sha] tree = self.repo[obj.tree] if tree is None: raise KeyError('Bad reference: %s' % ref) if path is None: path = self.root else: # check if path and self.root are same if not os.path.samefile(path, self.root): # if not, we need the path's tree # (a sub-tree of the commit tree) tree = self._obj_from_tree(tree, path) # write the tree self._write_tree_to_wt(tree, path) def cmd(self, cmd): """ Run a raw git command from the shell and return any output. Unlike other methods (which depend on Dulwich's git reimplementation and not git itself), this is dependent on the git shell command. The given git subcommand and arguments are prefixed with ``git`` and run through the subprocess module. To maintain the class's indifference to the current working directory, we also prepend the ``--git-dir`` and ``--work-tree`` arguments. :param cmd: A list of command-line arguments (anything the subprocess module will take). :return: a string containing the command's output. **Usage** (output has been truncated for brevity): >>> repo.cmd(['checkout', '-q', 'master']) >>> repo.cmd(['commit', '-q', '-a', '-m', 'Initial Commit']) >>> repo.cmd(['remote', '-v']) "origin [email protected]:hopper.git (fetch)\\n\\n origin ..." >>> repo.cmd(['log']) "commit 68a116eaee458607a3a9cf852df4f358a02bdb92\\nAuthor: Ni..." As you can see, it doesn't do any parsing of the output. It's available for times when the other methods don't get the job done. """ if not type(cmd) is list: raise TypeError('cmd must be a list') git_dir = os.path.join(self.root, '.git') prefix = ['git', '--git-dir', git_dir, '--work-tree', self.root] # It would be nice to use check_output() here, but it's 2.7+ return subprocess.Popen(prefix + cmd, stdout=subprocess.PIPE).communicate()[0] def commit(self, all=False, **kwargs): """ Commit the changeset to the repository. Equivalent to the `git commit` command. This method does a commit; use the ``commits`` method to retrieve one or more commits. Uses ``dulwich.objects.BaseRepo.do_commit()``, see that for params. At minimum, you need to provide **committer** and **message**. Everything else will be defaulted. :param all: commit all modified files that are already being tracked. :param \*\*kwargs: the commit attributes (e.g. committer, message, etc.). Again, see the underlying dulwich method. """ if all: # add all changes (to already tracked files) self.add(all=True, add_new_files=False) # pass the kwargs to dulwich, get the returned commit id. commit_id = self.repo.do_commit(**kwargs) # return the Commit object (instead of the id, which is less useful). return self.repo[commit_id] def commits(self, ref=None, n=10): """ Return up to n-commits down from a ref (branch, tag, commit), or if no ref given, down from the HEAD. If you just want a single commit, it may be cleaner to use the ``object`` method. :param ref: a branch, tag (not yet), or commit SHA to use as a start point. :param n: the maximum number of commits to return. If fewer matching commits exist, only they will be returned. :return: a list of ``dulwich.objects.Commit`` objects. **Usage**: >>> repo.commits() [<Commit 6f50a9bcd25ddcbf21919040609a9ad3c6354f1c>, <Commit 6336f47615da32d520a8d52223b9817ee50ca728>] >>> repo.commits()[0] == repo.head() True >>> repo.commits(n=1) [<Commit 6f50a9bcd25ddcbf21919040609a9ad3c6354f1c>] >>> repo.commits('6336f47615da32d520a8d52223b9817ee50ca728', n=1) [<Commit 6336f47615da32d520a8d52223b9817ee50ca728>] """ start_point = self.head().id if ref is not None: start_point = self._resolve_ref(ref) return self.repo.revision_history(start_point)[:n] def diff(self, a, b=None, path=None): """ Return a diff of commits a and b. :param a: a commit identifier. :param b: a commit identifier. Defaults to HEAD. :param path: a path to a file or directory to diff, relative to the repo root. Defaults to the entire tree. """ if not os.path.isfile(os.path.join(self.root, path)): raise NotImplementedError('Specify a file path for now') return self._diff_file(path, a, b) def head(self): """Return the HEAD commit or raise an error.""" # It seems best to make this a function so we don't have to # set and continually update it. try: return self.repo['HEAD'] except KeyError: # The HEAD will be missing before the repo is committed to. raise NoHeadSet def is_dirty(self): """Return True if there are uncommitted changes to the repository.""" new, modified, deleted = self.status() if new or modified or deleted: return True return False def object(self, sha): """ Retrieve an object from the repository. :param sha: the 40-byte hex-rep of the object's SHA1 identifier. """ return self.repo[sha] def status(self, from_path=None): """ Compare the working directory with HEAD. :param from_path: show changes within this path, which must be a file or directory relative to the repo. :return: a tuple containing three lists: new, modified, deleted """ # TODO: also compare the index and HEAD, or the index and WT # use from_path if set, otherwise root. if from_path is not None: from_path = os.path.join(self.root, from_path) if not os.path.exists(from_path): raise OSError('from_path does not exist.') path = from_path else: path = self.root # store changes in dictionary changes = {} changes['new'] = [] changes['modified'] = [] changes['deleted'] = [] # path is a file if os.path.isfile(path): status = self._file_status(path) if status == FILE_IS_NEW: changes['new'].append(path) elif status == FILE_IS_MODIFIED: changes['modified'].append(path) elif status == FILE_IS_DELETED: changes['deleted'].append(path) # path is a directory elif os.path.isdir(path): for directory, dirnames, filenames in os.walk(path): if '.git' in dirnames: dirnames.remove('.git') for f in filenames: fpath = os.path.relpath(os.path.join(directory, f), self.root) status = self._file_status(fpath) if status == FILE_IS_NEW: changes['new'].append(fpath) elif status == FILE_IS_MODIFIED: changes['modified'].append(fpath) elif status == FILE_IS_DELETED: changes['deleted'].append(fpath) return changes['new'], changes['modified'], changes['deleted'] def tag(self, name, ref=None): """ Create a tag. :param name: name of the new tag (e.g. 'v1.0' or '1.0.6') :param ref: a commit ref to tag, defaults to HEAD. """ # TODO: display tags attached to HEAD when no args. if ref is None: ref = self.head().id ref = self._resolve_ref(ref) self.repo.refs['refs/tags/%s' % name] = ref def tree(self, sha=None): """ Return the tree with given SHA, or if no SHA given, return the HEAD commit's tree. Raise an error if an object matches the SHA, but is not a tree. :param sha: tree reference. Note that a commit reference would not work. To get a commit's tree, just provide ``c.tree``, which contains the SHA we need. """ if sha is None: obj = self.repo[self.head().tree] else: obj = self.repo[sha] if type(obj) is Tree: return obj else: raise NotTreeError('Object is not a Tree') def _file_status(self, path, ref=None): """ Checks the status of a file in the working tree relative to a commit (usually HEAD). Statuses include: new, modified, and deleted. These statuses are conveyed as constants:: FILE_IS_UNCHANGED = 0 FILE_IS_NEW = 1 FILE_IS_MODIFIED = 2 FILE_IS_DELETED = 3 :param path: file path relative to the repo :param ref: optional ref to compare the WT with, default is HEAD. :return: status constant """ full_path = os.path.join(self.root, path) in_work_tree = os.path.exists(full_path) in_tree = self._file_in_tree(path) # new if not in_tree and in_work_tree: return FILE_IS_NEW # deleted elif in_tree and not in_work_tree: return FILE_IS_DELETED # modified elif in_tree and in_work_tree and self._file_is_modified(path): return FILE_IS_MODIFIED # unchanged elif in_tree and in_work_tree: return FILE_IS_UNCHANGED # does not exist (at least in our 2-tree world) else: raise KeyError('Path not found in either tree.') def _file_is_modified(self, path, ref=None): """ Returns True if the current file (in the WT) has been modified from the blob in the commit's tree, False otherwise. :param path: path to the file relative to the repository root. :param ref: optional ref to compare the WT with, default is HEAD. This returns False for new files (not present in the tree). If this is unexpected, just call ``_file_in_tree`` first. It assumes that the given path does exist. Just expect an OSError if it doesn't. """ # handle no head scenario when this gets called before first commit try: self.head() except NoHeadSet: return False # get the tree tree = self.repo[self.head().tree] # get the blob from the tree blob1 = self._obj_from_tree(tree, path) if type(blob1) is not Blob: return False # make a second blob from the current file with open(os.path.join(self.root, path), 'r') as fp: blob2 = Blob.from_string(fp.read()) # are the two blobs equivalent? # if their contents are the same they should be... # calls dulwich.objects.ShaFile.__eq__, which just compares SHAs return blob1 != blob2 def _file_in_tree(self, path, ref=None): """ Returns True if the file corresponds to a blob in the HEAD commit's tree, False otherwise. :param path: path to the file relative to the repository root. :param ref: optional ref to compare the WT with, default is HEAD. """ # handle no head scenario when this gets called before first commit try: self.head() except NoHeadSet: return False # get the tree tree = self.repo[self.head().tree] if self._obj_from_tree(tree, path) is not None: return True return False def _apply_to_tree(self, tree, f, path=None): """ Walk a tree recursively and apply function, f, to each entry :param tree: a dulwich.objects.Tree object :param f: function that will be called with each entry. :param path: if provided, the path relative to the repository will be included in the function call. """ if type(tree) is not Tree: raise NotTreeError for entry in tree.iteritems(): f(entry, path) if path else f(entry) obj = self.repo[entry.sha] if type(obj) is Tree: new_path = os.path.join(path, f) if path else None self._apply_to_tree(obj, f, new_path) def _obj_from_tree(self, tree, path): """ Walk a tree recursively to retrieve and return a blob or sub-tree from the given path, or return None if one does not exist. :param tree: a dulwich.objects.Tree object. :param path: path relative to the repository root. :return: Tree object, Blob object, or None if the path could not be found. For example, providing ``hopper/git.py`` would return the ``git.py`` blob within the ``hopper`` sub-tree. """ if type(tree) is not Tree: raise NotTreeError('Object is not a tree') # remove trailing slashes from path (so basename doesn't return '') if path[-1] == os.sep: path = path[:-1] # we need the head of the path, which is either the file itself or a # directory. head = path.split(os.sep)[0] if len(head) > 1: # clip head from path for recursion new_path = os.sep.join(path.split(os.sep)[1:]) for entry in tree.iteritems(): # these are dulwich.objects.TreeEntry objects if entry.path == head: # get the Tree or Blob. obj = self.repo[entry.sha] # return if we're at the right path if head == path: return obj # otherwise recurse if it's a Tree elif type(obj) is Tree: return self._obj_from_tree(obj, new_path) # if we get here the path wasn't there. return None def _write_tree_to_wt(self, tree, basepath): """ Walk a tree recursively and write each blob's data to the working tree. :param tree: a dulwich.objects.Tree object. :param basepath: blob data is written to: ``os.path.join(basepath, blob_path)``. Recursive calls will append the sub-tree name to the original call. """ if type(tree) is not Tree: raise NotTreeError('Object is not a tree') for entry in tree.iteritems(): obj = self.repo[entry.sha] if type(obj) is Blob: path = os.path.join(basepath, entry.path) with open(path, 'wb') as fp: fp.write(obj.data) elif type(obj) is Tree: new_basepath = os.path.join(basepath, entry.path) self._write_tree_to_wt(obj, new_basepath) def _resolve_ref(self, ref): """ Resolve a reference to a commit SHA. :param ref: branch, tag, commit reference. :return: a commit SHA. :raises KeyError: if ref doesn't point to a commit. :raises TypeError: if ref is not a string. """ # order: branch -> tag -> commit # (tag and branch can have same name, git assumes branch) if type(ref) is not str: raise TypeError('ref must be a string') # dulwich.Repo.refs keys the full name # (i.e. 'refs/heads/master') for branches and tags branch = _expand_branch_name(ref) tag = _expand_tag_name(ref) # branch? if branch in self.repo.refs: # get the commit SHA that the branch points to return self.repo[branch].id # tag? elif tag in self.repo.refs: return self.repo[tag].id # commit? else: obj = self.repo[ref] if type(obj) is Commit: return obj.id else: raise KeyError('Bad reference: %s' % ref) def _diff_file(self, path, a, b=None, html=False): """ Use difflib to compare a file between two commits, or a single commit and the working tree. :param a: ref to commit a. :param b: ref to commit b, defaults to the working tree. :param path: path to file, relative to repo root. :param html: format using difflib.HtmlDiff. :raise NotBlobError: if path wasn't present in both trees. """ # resolve commit a = self._resolve_ref(a) # get the trees tree1 = self.repo[self.repo[a].tree] # get the blob blob1 = self._obj_from_tree(tree1, path) # set data or empty string (meaning no blob at path) data1 = blob1.data if type(blob1) is Blob else '' if b is None: with open(os.path.join(self.root, path), 'r') as fp: data2 = fp.read() else: b = self._resolve_ref(b) tree2 = self.repo[self.repo[b].tree] blob2 = self._obj_from_tree(tree2, path) data2 = blob2.data if type(blob2) is Blob else '' # if both blobs were missing => bad path if type(blob1) is not Blob and type(blob2) is not Blob: raise NotBlobError( 'Path did not point to a blob in either tree') diff = list( difflib.context_diff(data1.splitlines(), data2.splitlines())) return '\n'.join(diff)
def setUp(self): TestCaseWithTransport.setUp(self) self.remote_real = GitRepo.init('remote', mkdir=True) self.remote_url = 'git://%s/' % os.path.abspath(self.remote_real.path) self.permit_url(self.remote_url)
def open_repo(path_or_repo): """Open an argument that can be a repository or a path for a repository.""" if isinstance(path_or_repo, BaseRepo): return path_or_repo return Repo(path_or_repo)
def setUp(self): super(PorcelainTestCase, self).setUp() repo_dir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, repo_dir) self.repo = Repo.init(repo_dir)
def setup(self): super(TestGitFileSystem, self).setup() Repo.init(self.colpath) from radicale.storage import filesystem filesystem.GIT_REPOSITORY = Repo(self.colpath)
# not really pull request, but just an example from dulwich.repo import Repo r = Repo('.') print(r.head()) c = r[r.head()] print(c) print("Message of the commit" + str(c.message))
def simple_commit_a(self): r = GitRepo.init('.') self.build_tree(['a']) r.stage(["a"]) return r.do_commit(b"a", committer=b"Somebody <*****@*****.**>")
from dulwich.repo import Repo # make sure we're starting fresh shutil.rmtree("demo_repo", ignore_errors=True) # the "lowest level" object in Git is probably the "blob"; it is just # some data. blob = Blob.from_string(b"Blobfish are people too\n") print("Created blob with hash: {}".format(blob.sha().hexdigest())) # if we actually want to store the Blob somewhere, we need a Git # database (which is the ".git/objects/*" directories and files in a # repository) repo = Repo.init("demo_repo", mkdir=True) print("Repository: {}".format(repo)) # this git database is called the "object store"; dulwich has an # abstraction of this store = repo.object_store print("Object store: {}".format(store)) store.add_object(blob) print("Added one object") # notes: # - single object file in .git/objects now # - "git cat-file -p <hash>": see our file! # - file is compressed (hexdump -C .git/objects/*) # now lets create a real "tree" object so we can give our Blob a