def method_rsync(log, src, base, dst, options=[], excludes=[]): run = log.pipe_powered(subprocess.run, stdout=logging.INFO, stderr=logging.WARNING) run(['rsync', '-rvt', '--partial', '--del', '--delete-excluded'] + (['--copy-dest', base] if os.path.isdir(base) else []) + sum([['--exclude', e] for e in excludes], []) + options + [src, dst], check=True)
def method_reposync(log, src, base, dst, arches=['noarch', 'x86_64'], source='auto', metadata='download', options=[], excludes=[]): if source == 'auto': source = '/source' in src or '/SRPM' in src excludes = [e[:-4] if e.endswith('.rpm') else e for e in excludes] repo_desc_for_mirroring = textwrap.dedent(f''' [repo] baseurl = {src} name = repo enabled = 1 gpgcheck = 0 ''') repodir = temp.disappearing_dir() with open(os.path.join(repodir, f'whatever.repo'), 'w') as f: f.write(repo_desc_for_mirroring) run = log.pipe_powered(subprocess.run, stdout=logging.INFO, stderr=logging.WARNING) run([ 'dnf', f'--setopt=reposdir={repodir}', 'reposync', '--norepopath', f'--download-path={dst}', '--repoid=repo', '--delete', '--remote-time' ] + [f'--arch={arch}' for arch in arches] + (['--download-metadata'] if metadata != 'generate' else []) + (['--source'] if source else []) + (['--exclude=' + ','.join(excludes)] if excludes else []) + options, check=True) run = log.pipe_powered( subprocess.run, # either too silent or too noisy =/ stdout=logging.INFO, stderr=logging.INFO) createrepo_c_options = ['-v', '--error-exit-val', '--ignore-lock'] if metadata == 'regenerate': log.info('regenerating metadata...') run(['createrepo_c'] + createrepo_c_options + ['--update', dst], check=True) elif metadata == 'generate': log.info('generating metadata from scratch...') run(['createrepo_c'] + createrepo_c_options + [dst], check=True)
def deduplicate(log, *subpath, timeout=None): log.info('locking the deduplication db...') with lock.Lock(path.saviour('.duperemove.hashfile-lock'), timeout=timeout): log.info('deduplicating...') run = log.pipe_powered(subprocess.run, stdout=logging.INFO, stderr=logging.WARNING) r = run(['duperemove', '--hashfile', path.saviour('.duperemove.hashfile'), '-hdr', path.saviour('_', *subpath)]) assert r.returncode in (0, 22) # nothing to deduplicate
def method_command(log, src, base, dst, command='false', reuse=True): if not reuse: fingertip.util.log.info(f'removing {dst}...') _remove(dst) env = os.environ.copy() env['SRC'], env['BASE'], env['DST'] = src, base, dst run = log.pipe_powered(subprocess.run, stdout=logging.INFO, stderr=logging.WARNING) log.info(command.replace('$SRC', src).replace('$BASE', base) .replace('$DST', dst)) run(command, shell=True, cwd=os.path.dirname(dst), env=env, check=True)
def _deduplicate(log, db_name, resource_name, timeout=None): log.info(f'locking the deduplication db {db_name}...') hashfilesdir = path.saviour('.duperemove', 'hashfiles') if not os.path.exists(hashfilesdir): os.makedirs(hashfilesdir) os.system(f'chattr +C {hashfilesdir} || true') db_file = path.saviour('.duperemove', 'hashfiles', db_name) db_lock = path.saviour('.duperemove', 'locks', db_name, makedirs=True) with lock.Lock(db_lock, timeout=timeout): log.info(f'deduplicating {resource_name} ({db_name})...') run = log.pipe_powered(subprocess.run, stdout=logging.INFO, stderr=logging.WARNING) r = run([ 'duperemove', '--dedupe-options=nofiemap', '--io-threads=2', '--cpu-threads=2', '--hashfile', db_file, '-hdr', path.saviour('_', resource_name, 'data') ]) assert r.returncode in (0, 22) # nothing to deduplicate
def validate_rpm_repository(log, _unused_src, dst): repo_desc_for_mirroring = textwrap.dedent(f''' [repo] baseurl = {dst} name = repo enabled = 1 gpgcheck = 0 ''') repodir = temp.disappearing_dir() with open(os.path.join(repodir, f'whatever.repo'), 'w') as f: f.write(repo_desc_for_mirroring) run = log.pipe_powered(subprocess.run, stdout=logging.DEBUG, stderr=logging.WARNING) run([ 'dnf', f'--setopt=reposdir={repodir}', '--repoid=repo', '--refresh', '--setopt=skip_if_unavailable=0', 'makecache' ], check=True)
def method_reposync(log, src, base, dst, arches=['noarch', 'x86_64'], source='auto', options=[]): if source == 'auto': source = '/source' in src or '/SRPM' in src repo_desc_for_mirroring = textwrap.dedent(f''' [repo] baseurl = {src} name = repo enabled = 1 gpgcheck = 0 ''') repodir = temp.disappearing_dir() with open(os.path.join(repodir, f'whatever.repo'), 'w') as f: f.write(repo_desc_for_mirroring) run = log.pipe_powered(subprocess.run, stdout=logging.INFO, stderr=logging.WARNING) run(['dnf', f'--setopt=reposdir={repodir}', 'reposync', f'--download-path={dst}', '--norepopath', '--download-metadata', '--delete', '--repoid=repo'] + [f'--arch={arch}' for arch in arches] + options + (['--source'] if source else []), check=True)