def method_rsync(log, src, base, dst, options=[], excludes=[]):
run = log.pipe_powered(subprocess.run,
stdout=logging.INFO, stderr=logging.WARNING)
run(['rsync', '-rvt', '--partial', '--del', '--delete-excluded'] +
(['--copy-dest', base] if os.path.isdir(base) else []) +
sum([['--exclude', e] for e in excludes], []) + options +
[src, dst], check=True)
def method_reposync(log,
src,
base,
dst,
arches=['noarch', 'x86_64'],
source='auto',
metadata='download',
options=[],
excludes=[]):
if source == 'auto':
source = '/source' in src or '/SRPM' in src
excludes = [e[:-4] if e.endswith('.rpm') else e for e in excludes]
repo_desc_for_mirroring = textwrap.dedent(f'''
[repo]
baseurl = {src}
name = repo
enabled = 1
gpgcheck = 0
''')
repodir = temp.disappearing_dir()
with open(os.path.join(repodir, f'whatever.repo'), 'w') as f:
f.write(repo_desc_for_mirroring)
run = log.pipe_powered(subprocess.run,
stdout=logging.INFO,
stderr=logging.WARNING)
run([
'dnf', f'--setopt=reposdir={repodir}', 'reposync', '--norepopath',
f'--download-path={dst}', '--repoid=repo', '--delete', '--remote-time'
] + [f'--arch={arch}' for arch in arches] +
(['--download-metadata'] if metadata != 'generate' else []) +
(['--source'] if source else []) +
(['--exclude=' + ','.join(excludes)] if excludes else []) + options,
check=True)
run = log.pipe_powered(
subprocess.run, # either too silent or too noisy =/
stdout=logging.INFO,
stderr=logging.INFO)
createrepo_c_options = ['-v', '--error-exit-val', '--ignore-lock']
if metadata == 'regenerate':
log.info('regenerating metadata...')
run(['createrepo_c'] + createrepo_c_options + ['--update', dst],
check=True)
elif metadata == 'generate':
log.info('generating metadata from scratch...')
run(['createrepo_c'] + createrepo_c_options + [dst], check=True)
def deduplicate(log, *subpath, timeout=None):
log.info('locking the deduplication db...')
with lock.Lock(path.saviour('.duperemove.hashfile-lock'), timeout=timeout):
log.info('deduplicating...')
run = log.pipe_powered(subprocess.run,
stdout=logging.INFO, stderr=logging.WARNING)
r = run(['duperemove',
'--hashfile', path.saviour('.duperemove.hashfile'),
'-hdr', path.saviour('_', *subpath)])
assert r.returncode in (0, 22) # nothing to deduplicate
def method_command(log, src, base, dst, command='false', reuse=True):
if not reuse:
fingertip.util.log.info(f'removing {dst}...')
_remove(dst)
env = os.environ.copy()
env['SRC'], env['BASE'], env['DST'] = src, base, dst
run = log.pipe_powered(subprocess.run,
stdout=logging.INFO, stderr=logging.WARNING)
log.info(command.replace('$SRC', src).replace('$BASE', base)
.replace('$DST', dst))
run(command, shell=True, cwd=os.path.dirname(dst), env=env,
check=True)
def _deduplicate(log, db_name, resource_name, timeout=None):
log.info(f'locking the deduplication db {db_name}...')
hashfilesdir = path.saviour('.duperemove', 'hashfiles')
if not os.path.exists(hashfilesdir):
os.makedirs(hashfilesdir)
os.system(f'chattr +C {hashfilesdir} || true')
db_file = path.saviour('.duperemove', 'hashfiles', db_name)
db_lock = path.saviour('.duperemove', 'locks', db_name, makedirs=True)
with lock.Lock(db_lock, timeout=timeout):
log.info(f'deduplicating {resource_name} ({db_name})...')
run = log.pipe_powered(subprocess.run,
stdout=logging.INFO,
stderr=logging.WARNING)
r = run([
'duperemove', '--dedupe-options=nofiemap', '--io-threads=2',
'--cpu-threads=2', '--hashfile', db_file, '-hdr',
path.saviour('_', resource_name, 'data')
])
assert r.returncode in (0, 22) # nothing to deduplicate
def validate_rpm_repository(log, _unused_src, dst):
repo_desc_for_mirroring = textwrap.dedent(f'''
[repo]
baseurl = {dst}
name = repo
enabled = 1
gpgcheck = 0
''')
repodir = temp.disappearing_dir()
with open(os.path.join(repodir, f'whatever.repo'), 'w') as f:
f.write(repo_desc_for_mirroring)
run = log.pipe_powered(subprocess.run,
stdout=logging.DEBUG,
stderr=logging.WARNING)
run([
'dnf', f'--setopt=reposdir={repodir}', '--repoid=repo', '--refresh',
'--setopt=skip_if_unavailable=0', 'makecache'
],
check=True)
def method_reposync(log, src, base, dst,
arches=['noarch', 'x86_64'], source='auto', options=[]):
if source == 'auto':
source = '/source' in src or '/SRPM' in src
repo_desc_for_mirroring = textwrap.dedent(f'''
[repo]
baseurl = {src}
name = repo
enabled = 1
gpgcheck = 0
''')
repodir = temp.disappearing_dir()
with open(os.path.join(repodir, f'whatever.repo'), 'w') as f:
f.write(repo_desc_for_mirroring)
run = log.pipe_powered(subprocess.run,
stdout=logging.INFO, stderr=logging.WARNING)
run(['dnf', f'--setopt=reposdir={repodir}', 'reposync',
f'--download-path={dst}', '--norepopath',
'--download-metadata', '--delete', '--repoid=repo'] +
[f'--arch={arch}' for arch in arches] + options +
(['--source'] if source else []),
check=True)
