def discover(cls, start='.'):
"""Iterate parent directories to discover a repository
Return a Repo object for the first parent directory that looks like a
Git repository.
:param start: The directory to start discovery from (defaults to '.')
"""
remaining = True
path = os.path.abspath(start)
while remaining:
try:
return cls(path)
except NotGitRepository:
path, remaining = os.path.split(path)
raise NotGitRepository("No git repository was found at %(path)s" %
dict(path=start))
def _http_request(self, url, headers={}, data=None):
parsed = urlparse.urlparse(url)
if parsed.username is not None:
url = url.replace(
'{0}:{1}@'.format(parsed.username, parsed.password), '')
req = urllib2.Request(url, headers=headers, data=data)
if parsed.username is not None:
req.add_header(
'Authorization',
b'Basic ' + base64.b64encode(parsed.username.encode() + b':' +
parsed.password.encode()))
try:
resp = self.opener.open(req)
except urllib2.HTTPError as e:
if e.code == 404:
raise NotGitRepository()
if e.code != 200:
raise GitProtocolError("unexpected http response %d" % e.code)
return resp
def __init__(self, path):
self.path = path
if not isinstance(path, bytes):
self._path_bytes = path.encode(sys.getfilesystemencoding())
else:
self._path_bytes = path
if os.path.isdir(os.path.join(self._path_bytes, b'.git', OBJECTDIR)):
self.bare = False
self._controldir = os.path.join(self._path_bytes, b'.git')
elif (os.path.isdir(os.path.join(self._path_bytes, OBJECTDIR))
and os.path.isdir(os.path.join(self._path_bytes, REFSDIR))):
self.bare = True
self._controldir = self._path_bytes
elif (os.path.isfile(os.path.join(self._path_bytes, b'.git'))):
import re
with open(os.path.join(self._path_bytes, b'.git'), 'rb') as f:
_, gitdir = re.match(b'(gitdir: )(.+$)', f.read()).groups()
self.bare = False
self._controldir = os.path.join(self._path_bytes, gitdir)
else:
raise NotGitRepository("No git repository was found at %(path)s" %
dict(path=path))
object_store = DiskObjectStore(
os.path.join(self.controldir(), OBJECTDIR))
refs = DiskRefsContainer(self.controldir())
BaseRepo.__init__(self, object_store, refs)
self._graftpoints = {}
graft_file = self.get_named_file(os.path.join(b'info', b'grafts'))
if graft_file:
with graft_file:
self._graftpoints.update(parse_graftpoints(graft_file))
graft_file = self.get_named_file(b'shallow')
if graft_file:
with graft_file:
self._graftpoints.update(parse_graftpoints(graft_file))
self.hooks['pre-commit'] = PreCommitShellHook(self.controldir())
self.hooks['commit-msg'] = CommitMsgShellHook(self.controldir())
self.hooks['post-commit'] = PostCommitShellHook(self.controldir())
def __init__(self, root):
if os.path.isdir(os.path.join(root, ".git", OBJECTDIR)):
self.bare = False
self._controldir = os.path.join(root, ".git")
elif (os.path.isdir(os.path.join(root, OBJECTDIR)) and
os.path.isdir(os.path.join(root, REFSDIR))):
self.bare = True
self._controldir = root
elif (os.path.isfile(os.path.join(root, ".git"))):
import re
f = open(os.path.join(root, ".git"), 'r')
try:
_, path = re.match('(gitdir: )(.+$)', f.read()).groups()
finally:
f.close()
self.bare = False
self._controldir = os.path.join(root, path)
else:
raise NotGitRepository(
"No git repository was found at %(path)s" % dict(path=root)
)
self.path = root
object_store = DiskObjectStore(os.path.join(self.controldir(),
OBJECTDIR))
refs = DiskRefsContainer(self.controldir())
BaseRepo.__init__(self, object_store, refs)
self._graftpoints = {}
graft_file = self.get_named_file(os.path.join("info", "grafts"))
if graft_file:
with graft_file:
self._graftpoints.update(parse_graftpoints(graft_file))
graft_file = self.get_named_file("shallow")
if graft_file:
with graft_file:
self._graftpoints.update(parse_graftpoints(graft_file))
self.hooks['pre-commit'] = PreCommitShellHook(self.controldir())
self.hooks['commit-msg'] = CommitMsgShellHook(self.controldir())
self.hooks['post-commit'] = PostCommitShellHook(self.controldir())
def _http_request(self,
url,
headers=None,
data=None,
allow_compression=False):
req_headers = self.session.headers.copy()
if headers is not None:
req_headers.update(headers)
if allow_compression:
req_headers["Accept-Encoding"] = "gzip"
else:
req_headers["Accept-Encoding"] = "identity"
if data:
resp = self.session.post(url, headers=req_headers, data=data)
else:
resp = self.session.get(url, headers=req_headers)
if resp.status_code == 404:
raise NotGitRepository()
if resp.status_code == 401:
raise HTTPUnauthorized(resp.headers.get("WWW-Authenticate"), url)
if resp.status_code == 407:
raise HTTPProxyUnauthorized(resp.headers.get("Proxy-Authenticate"),
url)
if resp.status_code != 200:
raise GitProtocolError("unexpected http resp %d for %s" %
(resp.status_code, url))
# Add required fields as stated in AbstractHttpGitClient._http_request
resp.content_type = resp.headers.get("Content-Type")
resp.redirect_location = ""
if resp.history:
resp.redirect_location = resp.url
read = BytesIO(resp.content).read
return resp, read
def __init__(self, root):
hidden_path = os.path.join(root, CONTROLDIR)
if os.path.isdir(os.path.join(hidden_path, OBJECTDIR)):
self.bare = False
self._controldir = hidden_path
elif (os.path.isdir(os.path.join(root, OBJECTDIR)) and
os.path.isdir(os.path.join(root, REFSDIR))):
self.bare = True
self._controldir = root
elif os.path.isfile(hidden_path):
self.bare = False
with open(hidden_path, 'r') as f:
path = read_gitfile(f)
self.bare = False
self._controldir = os.path.join(root, path)
else:
raise NotGitRepository(
"No git repository was found at %(path)s" % dict(path=root)
)
self.path = root
object_store = DiskObjectStore(os.path.join(self.controldir(),
OBJECTDIR))
refs = DiskRefsContainer(self.controldir())
BaseRepo.__init__(self, object_store, refs)
self._graftpoints = {}
graft_file = self.get_named_file(os.path.join("info", "grafts"))
if graft_file:
with graft_file:
self._graftpoints.update(parse_graftpoints(graft_file))
graft_file = self.get_named_file("shallow")
if graft_file:
with graft_file:
self._graftpoints.update(parse_graftpoints(graft_file))
self.hooks['pre-commit'] = PreCommitShellHook(self.controldir())
self.hooks['commit-msg'] = CommitMsgShellHook(self.controldir())
self.hooks['post-commit'] = PostCommitShellHook(self.controldir())
def _discover_references(self, service, url):
assert url[-1] == "/"
url = urlparse.urljoin(url, "info/refs")
headers = {}
if self.dumb != False:
url += "?service=%s" % service
headers["Content-Type"] = "application/x-%s-request" % service
req = urllib2.Request(url, headers=headers)
resp = self._perform(req)
if resp.getcode() == 404:
raise NotGitRepository()
if resp.getcode() != 200:
raise GitProtocolError("unexpected http response %d" %
resp.getcode())
self.dumb = (not resp.info().gettype().startswith("application/x-git-"))
proto = Protocol(resp.read, None)
if not self.dumb:
# The first line should mention the service
pkts = list(proto.read_pkt_seq())
if pkts != [('# service=%s\n' % service)]:
raise GitProtocolError(
"unexpected first line %r from smart server" % pkts)
return self._read_refs(proto)
def open_repository(self, path):
logger.debug('opening repository at %s', path)
abspath = os.path.abspath(os.path.join(self.root, path)) + "/"
if not abspath.startswith(self.root):
raise NotGitRepository("Invalid path %r" % path)
return Repo(abspath)
class CommonGitLoaderNotFound:
@pytest.fixture(autouse=True)
def __inject_fixtures(self, mocker):
"""Inject required fixtures in unittest.TestCase class"""
self.mocker = mocker
@pytest.mark.parametrize(
"failure_exception",
[
GitProtocolError("Repository unavailable"), # e.g DMCA takedown
GitProtocolError("Repository not found"),
GitProtocolError("unexpected http resp 401"),
NotGitRepository("not a git repo"),
],
)
def test_load_visit_not_found(self, failure_exception):
"""Ingesting an unknown url result in a visit with not_found status"""
# simulate an initial communication error (e.g no repository found, ...)
mock = self.mocker.patch(
"swh.loader.git.loader.GitLoader.fetch_pack_from_origin")
mock.side_effect = failure_exception
res = self.loader.load()
assert res == {"status": "uneventful"}
assert_last_visit_matches(
self.loader.storage,
self.repo_url,
status="not_found",
type="git",
snapshot=None,
)
@pytest.mark.parametrize(
"failure_exception",
[
IOError,
ObjectFormatException,
OSError,
ValueError,
GitProtocolError,
],
)
def test_load_visit_failure(self, failure_exception):
"""Failing during the fetch pack step result in failing visit"""
# simulate a fetch communication error after the initial connection
# server error (e.g IOError, ObjectFormatException, ...)
mock = self.mocker.patch(
"swh.loader.git.loader.GitLoader.fetch_pack_from_origin")
mock.side_effect = failure_exception("failure")
res = self.loader.load()
assert res == {"status": "failed"}
assert_last_visit_matches(
self.loader.storage,
self.repo_url,
status="failed",
type="git",
snapshot=None,
)
def open_repository(self, path):
full_path = os.path.normpath(os.path.join(self.root, path.lstrip("/")))
if not full_path.startswith(self.root + "/"):
raise NotGitRepository("Repository %s not under store" % path)
return GitRepo(full_path)