def __init__(self, url, *path_components, enough_to_have=None):
with Repo(url, *path_components, enough_to_have=enough_to_have) as r:
cache_path = path.downloads('git', *path_components, makedirs=True)
self.path = temp.disappearing_dir(os.path.dirname(cache_path),
path_components[-1])
self.self_destruct = False
git.Repo.clone_from(r.path, self.path)
git.Repo.__init__(self, self.path)
def hack_around_unpacking(uri, headers, wrong_content):
log.warning(f're-fetching correct content for {uri}')
r = requests.get(uri, headers=headers, stream=True, allow_redirects=False)
h = hashlib.sha256(wrong_content).hexdigest()
cachefile = path.downloads('fixups', h, makedirs=True)
if not os.path.exists(cachefile):
with path.wip(cachefile) as wip:
with open(wip, 'wb') as f:
shutil.copyfileobj(r.raw, f)
with open(cachefile, 'rb') as f:
return f.read()
def _get_requests_session(self, direct=False):
if not direct:
cache = cachecontrol.caches.FileCache(path.downloads('cache'))
sess = cachecontrol.CacheControl(requests.Session(), cache=cache)
else:
sess = requests.Session()
for uri, kwargs in self._mocks:
adapter = requests_mock.Adapter()
adapter.register_uri('HEAD', uri, **kwargs)
adapter.register_uri('GET', uri, **kwargs)
sess.mount(uri, adapter)
return sess
def _get_requests_session(self, direct=False):
if not direct:
kwargs = ({'filemode': 0o0660, 'dirmode': 0o0770}
if is_cache_group_writeable() else {})
cache = cachecontrol.caches.FileCache(path.downloads('cache'),
**kwargs)
sess = cachecontrol.CacheControl(requests.Session(), cache=cache)
else:
sess = requests.Session()
for uri, kwargs in self._mocks.items():
adapter = requests_mock.Adapter()
adapter.register_uri('HEAD', uri, **kwargs)
adapter.register_uri('GET', uri, **kwargs)
sess.mount(uri, adapter)
return sess
# Licensed under GNU General Public License v3 or later, see COPYING.
# Copyright (c) 2019 Red Hat, Inc., see CONTRIBUTORS.
import fasteners
import git
import os
import tarfile
from fingertip.util import log, path, temp
OFFLINE = os.getenv('FINGERTIP_OFFLINE', '0') != '0'
DIR = path.downloads('git')
class Repo(git.Repo):
def __init__(self, url, *path_components, enough_to_have=None):
self.url = url
self.path = os.path.join(DIR, *path_components)
lock_path = self.path + '-lock'
self.lock = fasteners.process_lock.InterProcessLock(lock_path)
self.lock.acquire()
if not os.path.exists(self.path):
log.info(f'cloning {url}...')
git.Repo.clone_from(url, self.path, mirror=True) # TODO: use bare
super().__init__(self.path)
else:
super().__init__(self.path)
update_not_needed = enough_to_have and (
enough_to_have in (t.name for t in self.tags)
or enough_to_have in (h.name for h in self.heads)
def __init__(self, url, *path_components, enough_to_have=None):
if not path_components:
path_components = [url.replace('/', '::')]
self.url = url
cache_path = path.downloads('git', *path_components, makedirs=True)
self.path = temp.disappearing_dir(os.path.dirname(cache_path),
path_components[-1])
lock_working_copy_path = self.path + '-lock'
lock_cache_path = cache_path + '-lock'
lock.Lock.__init__(self, lock_working_copy_path)
sources = saviour_sources()
self.self_destruct = False
with lock.Lock(lock_cache_path), lock.Lock(lock_working_copy_path):
cache_is_enough = False
if os.path.exists(cache_path):
try:
cr = git.Repo(cache_path)
cache_is_enough = (enough_to_have
and _has_rev(cr, enough_to_have))
except git.GitError as e:
log.error(f'something wrong with git cache {cache_path}')
log.error(str(e))
_remove(self.path)
for i, (source, cache) in enumerate(sources):
last_source = i == len(sources) - 1
if cache and cache_is_enough:
log.info(f'not re-fetching {url} from {source} '
f'because {enough_to_have} '
'is already present in cache')
git.Repo.clone_from(cache_path, self.path, mirror=True)
break
if source == 'local':
surl = path.saviour(url).replace('//', '/') # workaround
if not os.path.exists(surl) and not last_source:
continue
log.info(f'cloning {url} from local saviour mirror')
git.Repo.clone_from(surl, self.path, mirror=True)
break
elif source == 'direct':
surl = url
else:
surl = source + '/' + url
surl = 'http://' + surl if '://' not in source else surl
log.info(f'cloning {url} from {source} '
f'cache_exists={os.path.exists(cache_path)}...')
try:
# TODO: bare clone
# no harm in referencing cache, even w/o cached+
git.Repo.clone_from(surl,
self.path,
mirror=True,
dissociate=True,
reference_if_able=cache_path)
except git.GitError:
log.warning(f'could not clone {url} from {source}')
if last_source:
raise
continue
break
_remove(cache_path)
reflink.auto(self.path, cache_path)
git.Repo.__init__(self, self.path)
self.remotes[0].set_url(url)
self.self_destruct = True
def __init__(self, url, *path_components, enough_to_have=None):
assert path_components
self.url = url
cache_path = path.downloads('git', *path_components, makedirs=True)
cache_exists = os.path.exists(cache_path)
self.path = temp.disappearing_dir(os.path.dirname(cache_path),
path_components[-1])
lock_working_copy_path = self.path + '-lock'
lock_cache_path = cache_path + '-lock'
lock.Lock.__init__(self, lock_working_copy_path)
update_not_needed = None
sources = saviour_sources()
self.self_destruct = False
with lock.Lock(lock_cache_path), lock.Lock(lock_working_copy_path):
_remove(self.path)
for i, (source, cache) in enumerate(sources):
last_source = i == len(sources) - 1
if cache and cache_exists and update_not_needed is None:
cr = git.Repo(cache_path)
update_not_needed = enough_to_have and (
enough_to_have in (t.name for t in cr.tags) or
enough_to_have in (h.name for h in cr.heads) or
enough_to_have in (c.hexsha for c in cr.iter_commits())
# that's not all revspecs, but best-effort is fine
)
if update_not_needed:
log.info(f'not re-fetching {url} from {source} '
f'because {enough_to_have} '
'is already present in cache')
git.Repo.clone_from(cache_path, self.path, mirror=True)
break
if source == 'local':
surl = path.saviour(url).replace('//', '/') # workaround
if not os.path.exists(surl) and not last_source:
continue
log.info(f'cloning {url} from local saviour mirror')
git.Repo.clone_from(surl, self.path, mirror=True)
break
elif source == 'direct':
surl = url
else:
surl = source + '/' + url
surl = 'http://' + surl if '://' not in source else surl
log.info(f'cloning {url} from {source} '
f'cache_exists={cache_exists}...')
try:
# TODO: bare clone
# no harm in referencing cache, even w/o cached+
git.Repo.clone_from(surl, self.path, mirror=True,
dissociate=True,
reference_if_able=cache_path)
except git.GitError:
log.warning(f'could not clone {url} from {source}')
if last_source:
raise
continue
break
_remove(cache_path)
reflink.auto(self.path, cache_path)
git.Repo.__init__(self, self.path)
self.remotes[0].set_url(url)
self.self_destruct = True
