def _do_update(self, statepath: str, logger: Logger) -> bool: old_head = get_subprocess_output(['git', 'rev-parse', 'HEAD'], cwd=statepath, logger=logger).strip() run_subprocess([ 'timeout', str(self.fetch_timeout), 'git', 'fetch', '--progress', '--depth=1' ], cwd=statepath, logger=logger) run_subprocess( ['git', 'checkout'], cwd=statepath, logger=logger ) # needed for reset to not fail on changed sparse checkout self._setup_sparse_checkout(statepath, logger) run_subprocess(['git', 'reset', '--hard', 'origin/' + self.branch], cwd=statepath, logger=logger) run_subprocess(['git', 'reflog', 'expire', '--expire=0', '--all'], cwd=statepath, logger=logger) run_subprocess(['git', 'prune'], cwd=statepath, logger=logger) new_head = get_subprocess_output(['git', 'rev-parse', 'HEAD'], cwd=statepath, logger=logger).strip() if new_head == old_head: logger.log('HEAD has not changed: {}'.format(new_head)) return False logger.log('HEAD was updated from {} to {}'.format(old_head, new_head)) return True
def fetch(self, statepath: str, update: bool = True, logger: Logger = NoopLogger()) -> bool: if os.path.exists(statepath) and not update: logger.log('no update requested, skipping') return False args = [ '--info=stats2', '--archive', '--compress', '--delete', '--delete-excluded', '--safe-links', ] if self.fetch_timeout is not None: args += ['--timeout', str(self.fetch_timeout)] if self.rsync_include is not None: args += ['--include', self.rsync_include] if self.rsync_exclude is not None: args += ['--exclude', self.rsync_exclude] run_subprocess(['rsync'] + args + [self.url, statepath], logger) return True
def _do_fetch(self, statepath: str, logger: Logger) -> bool: run_subprocess([ 'timeout', str(self.fetch_timeout), 'svn', 'checkout', self.url, statepath ], logger=logger) return True
def _do_fetch(self, statepath: str, logger: Logger) -> bool: run_subprocess([ 'timeout', str(self.fetch_timeout), 'git', 'clone', '--progress', '--no-checkout', '--depth=1', '--branch', self.branch, self.url, statepath ], logger=logger) self._setup_sparse_checkout(statepath, logger) run_subprocess(['git', 'checkout'], cwd=statepath, logger=logger) return True
def _setup_sparse_checkout(self, statepath: str, logger: Logger) -> None: sparse_checkout_path = os.path.join(statepath, '.git', 'info', 'sparse-checkout') # We always enable sparse checkout, as it's harder to # properly disable sparse checkout and restore all files # than to leave it enabled with all files whitelisted # # See https://stackoverflow.com/questions/36190800/how-to-disable-sparse-checkout-after-enabled/36195275 run_subprocess(['git', 'config', 'core.sparsecheckout', 'true'], cwd=statepath, logger=logger) with open(sparse_checkout_path, 'w') as sparse_checkout_file: if self.sparse_checkout: for item in self.sparse_checkout: print(item, file=sparse_checkout_file) else: print('/*', file=sparse_checkout_file)
def _do_update(self, statepath: str, logger: Logger) -> bool: old_rev = get_subprocess_output( ['svn', 'info', '--show-item', 'revision', statepath], logger=logger).strip() run_subprocess( ['timeout', str(self.fetch_timeout), 'svn', 'up', statepath], logger=logger) new_rev = get_subprocess_output( ['svn', 'info', '--show-item', 'revision', statepath], logger=logger).strip() if new_rev == old_rev: logger.log('Revision has not changed: {}'.format(new_rev)) return False logger.log('Revision was updated from {} to {}'.format( old_rev, new_rev)) return True
def _do_fetch(self, statedir: AtomicDir, persdata: PersistentData, logger: Logger) -> bool: tarpath = os.path.join(statedir.get_path(), '.temporary.tar') headers = {} if persdata.get('last-modified'): headers['if-modified-since'] = persdata.get('last-modified') logger.log('using if-modified-since: {}'.format( headers['if-modified-since'])) logger.log('fetching {}'.format(self.url)) try: with open(tarpath, 'wb') as tarfile: response = save_http_stream(self.url, tarfile, headers=headers, timeout=self.fetch_timeout) except NotModifiedException: logger.log('got 403 not modified') return False # XXX: may be unportable, FreeBSD tar automatically handles compression type, # may not be the case on linuxes # XXX: this extracts tarball permissions, which is not desirable and it may # produce non-readable files and dirs (blackarch). GNU tar has --mode, BSD tar # lacks this. We should probably require GNU tar, and handle binary name which # may differ on BSD. run_subprocess( ['tar', '-x', '-z', '-f', tarpath, '-C', statedir.get_path()], logger) os.remove(tarpath) if response.headers.get('last-modified'): persdata['last-modified'] = response.headers['last-modified'] logger.log('storing last-modified: {}'.format( persdata['last-modified'])) return True
def _do_fetch(self, statedir: AtomicDir, persdata: PersistentData, logger: Logger) -> bool: tarpath = os.path.join(statedir.get_path(), '.temporary.tar') headers = {} if persdata.get('last-modified'): headers['if-modified-since'] = persdata.get('last-modified') logger.log('using if-modified-since: {}'.format( headers['if-modified-since'])) logger.log('fetching {}'.format(self.url)) try: with open(tarpath, 'wb') as tarfile: response = save_http_stream(self.url, tarfile, headers=headers, timeout=self.fetch_timeout) except NotModifiedException: logger.log('got 403 not modified') return False # XXX: may be unportable, FreeBSD tar automatically handles compression type, # may not be the case on linuxes run_subprocess( ['tar', '-x', '-z', '-f', tarpath, '-C', statedir.get_path()], logger) os.remove(tarpath) if response.headers.get('last-modified'): persdata['last-modified'] = response.headers['last-modified'] logger.log('storing last-modified: {}'.format( persdata['last-modified'])) return True