def do_fetch(self, statedir, logger):
tarpath = os.path.join(statedir, '.temporary.tar')
RunSubprocess([
'wget', '--timeout',
str(self.fetch_timeout), '--tries', '1', '-O', tarpath, self.url
], logger)
RunSubprocess(['tar', '-x', '-z', '-f', tarpath, '-C', statedir],
logger)
os.remove(tarpath)
def do_fetch(self, statedir, logger):
RunSubprocess([
'timeout',
str(self.fetch_timeout), 'git', 'clone', '--progress',
'--no-checkout', '--depth=1', '--branch', self.branch, self.url,
statedir
],
logger=logger)
self._setup_sparse_checkout(statedir, logger)
RunSubprocess(['git', 'checkout'], cwd=statedir, logger=logger)
def Fetch(self, statepath, update=True, logger=NoopLogger()):
if os.path.isdir(statepath) and not update:
logger.Log('no update requested, skipping')
return
with StateDir(statepath) as statedir:
tarpath = os.path.join(statedir, '.temporary.tar')
RunSubprocess(['wget', '--timeout=60', '-O', tarpath, self.url],
logger)
RunSubprocess(['tar', '-x', '-z', '-f', tarpath, '-C', statedir],
logger)
os.remove(tarpath)
def Fetch(self, statepath, update=True, logger=NoopLogger()):
if not os.path.isdir(statepath):
RunSubprocess([
'git', 'clone', '--progress', '--depth=1', self.url, statepath
],
logger=logger)
elif update:
RunSubprocess(['git', 'fetch', '--progress'],
cwd=statepath,
logger=logger)
RunSubprocess(['git', 'reset', '--hard', 'origin/' + self.branch],
cwd=statepath,
logger=logger)
else:
logger.Log('no update requested, skipping')
def Fetch(self, statepath, update=True, logger=NoopLogger()):
if os.path.isdir(statepath) and not update:
logger.Log('no update requested, skipping')
return
command = ['rsync', '--verbose', '--archive', '--compress', '--delete', '--delete-excluded', '--timeout=60', self.url, statepath]
RunSubprocess(command, logger)
def Fetch(self, statepath, update=True, logger=NoopLogger()):
if not os.path.isdir(statepath):
RunSubprocess([
"git", "clone", "--progress", "--depth=1", self.repository,
statepath
],
logger=logger)
elif update:
RunSubprocess(["git", "fetch", "--progress"],
cwd=statepath,
logger=logger)
RunSubprocess(["git", "reset", "--hard", "origin/master"],
cwd=statepath,
logger=logger)
else:
logger.Log("no update requested, skipping")
def Fetch(self, statepath, update=True, logger=NoopLogger()):
if not os.path.isdir(statepath):
RunSubprocess([
'git', 'clone', '--progress', '--no-checkout', '--depth=1',
'--branch', self.branch, self.url, statepath
],
logger=logger)
self.__SetupSparseCheckout(statepath, logger)
RunSubprocess(['git', 'checkout'], cwd=statepath, logger=logger)
elif update:
RunSubprocess(
['timeout', '10m', 'git', 'fetch', '--progress', '--depth=1'],
cwd=statepath,
logger=logger)
RunSubprocess(
['git', 'checkout'], cwd=statepath, logger=logger
) # needed for reset to not fail on changed sparse checkout
self.__SetupSparseCheckout(statepath, logger)
RunSubprocess(['git', 'reset', '--hard', 'origin/' + self.branch],
cwd=statepath,
logger=logger)
RunSubprocess(['git', 'reflog', 'expire', '--expire=0', '--all'],
cwd=statepath,
logger=logger)
RunSubprocess(['git', 'prune'], cwd=statepath, logger=logger)
else:
logger.Log('no update requested, skipping')
def do_update(self, statedir, logger):
RunSubprocess([
'timeout',
str(self.fetch_timeout), 'git', 'fetch', '--progress', '--depth=1'
],
cwd=statedir,
logger=logger)
RunSubprocess(
['git', 'checkout'], cwd=statedir, logger=logger
) # needed for reset to not fail on changed sparse checkout
self._setup_sparse_checkout(statedir, logger)
RunSubprocess(['git', 'reset', '--hard', 'origin/' + self.branch],
cwd=statedir,
logger=logger)
RunSubprocess(['git', 'reflog', 'expire', '--expire=0', '--all'],
cwd=statedir,
logger=logger)
RunSubprocess(['git', 'prune'], cwd=statedir, logger=logger)
def Fetch(self, statepath, update=True, logger=NoopLogger()):
if os.path.isdir(statepath) and not update:
logger.Log('no update requested, skipping')
return
if os.path.exists(statepath):
shutil.rmtree(statepath)
os.mkdir(statepath)
tempfile = os.path.join(statepath, '.temporary.tar')
try:
RunSubprocess(['wget', '-O', tempfile, self.url], logger)
RunSubprocess(['tar', '-x', '-z', '-f', tempfile, '-C', statepath],
logger)
os.remove(tempfile)
except:
if os.path.exists(statepath):
shutil.rmtree(statepath)
raise
def Fetch(self, statepath, update=True, logger=NoopLogger()):
if os.path.isdir(statepath) and not update:
logger.Log('no update requested, skipping')
return
try:
command = 'rsync --verbose --archive --compress --delete --delete-excluded {} {}'.format(
self.url, statepath)
RunSubprocess(command, logger, shell=True)
except:
if os.path.exists(statepath):
shutil.rmtree(statepath)
raise
def __SetupSparseCheckout(self, statepath, logger):
sparse_checkout_path = os.path.join(statepath, '.git', 'info', 'sparse-checkout')
# We always enable sparse checkout, as it's harder to
# properly disable sparse checkout and restore all files
# than to leave it enabled with all files whitelisted
#
# See https://stackoverflow.com/questions/36190800/how-to-disable-sparse-checkout-after-enabled/36195275
RunSubprocess(['git', 'config', 'core.sparsecheckout', 'true'], cwd=statepath, logger=logger)
with open(sparse_checkout_path, 'w') as sparse_checkout_file:
if self.sparse_checkout:
for item in self.sparse_checkout:
print(item, file=sparse_checkout_file)
else:
print('/*', file=sparse_checkout_file)
def Fetch(self, statepath, update=True, logger=NoopLogger()):
if os.path.isdir(statepath) and not update:
logger.Log('no update requested, skipping')
return
try:
command = [
'rsync', '--verbose', '--archive', '--compress', '--delete',
'--delete-excluded', self.url, statepath
]
RunSubprocess(command, logger)
except:
if os.path.exists(statepath):
shutil.rmtree(statepath)
raise
def Fetch(self, statepath, update=True, logger=NoopLogger()):
if os.path.isdir(statepath) and not update:
logger.Log('no update requested, skipping')
return
if os.path.exists(statepath):
shutil.rmtree(statepath)
os.mkdir(statepath)
try:
command = 'wget -O- "%s" | tar -xz -f- -C "%s"' % (self.url,
statepath)
RunSubprocess(command, logger, shell=True)
except:
if os.path.exists(statepath):
shutil.rmtree(statepath)
raise
def fetch(self, statepath, update=True, logger=NoopLogger()):
if os.path.exists(statepath) and not update:
logger.Log('no update requested, skipping')
return
args = [
'--verbose',
'--archive',
'--compress',
'--delete',
'--delete-excluded',
'--safe-links',
]
if self.fetch_timeout is not None:
args += ['--timeout', str(self.fetch_timeout)]
if self.rsync_include is not None:
args += ['--include', self.rsync_include]
if self.rsync_exclude is not None:
args += ['--exclude', self.rsync_exclude]
RunSubprocess(['rsync'] + args + [self.url, statepath], logger)
def do_update(self, statedir, logger):
RunSubprocess(
['timeout',
str(self.fetch_timeout), 'svn', 'up', statedir],
logger=logger)
def do_fetch(self, statedir, logger):
RunSubprocess([
'timeout',
str(self.fetch_timeout), 'svn', 'checkout', self.url, statedir
],
logger=logger)
def DoFetch(self, statepath, update, logger):
for source in self.sources:
command = "wget -O- \"%s\" | tar -xz -f- -C \"%s\"" % (source,
statepath)
RunSubprocess(command, logger, shell=True)
