def _remove(pkgnames, target_archs=[a for a in ARCHS if a != 'any']):
assert type(pkgnames) is list and pkgnames
assert not [None for s in pkgnames if not (type(s) is str)]
logger.info('starting remove %s for %s', pkgnames, target_archs)
if len(target_archs) == 1 and target_archs[0] == 'any':
target_archs = ARCHS
else:
assert 'any' not in target_archs
for arch in target_archs:
remove_pkgs = list()
basedir = Path('www') / arch
for fpath in basedir.iterdir():
if fpath.name.endswith(PKG_SUFFIX) and \
get_pkg_details_from_name(fpath.name).pkgname in pkgnames:
remove_pkgs.append(fpath)
if remove_pkgs:
logger.info("repo-remove: %s", repo_remove(remove_pkgs))
else:
logger.warning(f'Nothing to remove in {arch}')
archive_dir = Path('archive')
for fpath in archive_dir.iterdir():
nosigname = fpath.name[:-4] if fpath.name.endswith(
'.sig') else fpath.name
if nosigname.endswith(PKG_SUFFIX) and \
get_pkg_details_from_name(nosigname).pkgname in pkgnames:
throw_away(fpath)
logger.info('finished remove')
return True
def filter_old_pkg(fpaths, keep_new=1, archive=False, recycle=False):
'''
Accepts a list of paths (must be in the same dir)
return a tuple of list of paths
([new1, new2], [old1, old2])
packages are arranged from new to old, one by one.
new: pkga-v8, pkga-v7, pkgb-v5, pkgb-v4
old: pkga-v6, pkga-v5, pkgb-v3, pkgb-v2
(assume keep_new=2)
'''
if not fpaths:
return (list(), list())
assert type(fpaths) is list
for fpath in fpaths:
assert issubclass(type(fpath), os.PathLike) and \
fpath.name.endswith(PKG_SUFFIX)
assert not (archive and recycle)
assert not [None for fpath in fpaths if fpath.parent != fpaths[0].parent]
new_pkgs = list()
old_pkgs = list()
pkgs_vers = dict()
for fpath in fpaths:
pkg = get_pkg_details_from_name(fpath.name)
pkgs_vers.setdefault(pkg.pkgname + pkg.arch, list()).append(pkg)
for pkgname_arch in pkgs_vers:
family = pkgs_vers[pkgname_arch]
# new packages first
family = sorted(family, reverse=True)
if len(family) > keep_new:
new_pkgs += family[:keep_new]
old_pkgs += family[keep_new:]
else:
new_pkgs += family
for pkg in old_pkgs:
fullpath = fpaths[0].parent / pkg.fname
sigpath = fpaths[0].parent / f'{pkg.fname}.sig'
if archive:
archive_pkg(fullpath)
if sigpath.exists():
archive_pkg(sigpath)
elif recycle:
throw_away(fullpath)
if sigpath.exists():
throw_away(sigpath)
return (new_pkgs, old_pkgs)
def repo_remove(fpaths):
assert type(fpaths) is list
assert not [None for fpath in fpaths if fpath.parent != fpaths[0].parent]
for fpath in fpaths:
assert issubclass(type(fpath), os.PathLike) and \
fpath.name.endswith(PKG_SUFFIX)
dbpath = fpaths[0].parent / f'{REPO_NAME}.db.tar.gz'
for fpath in fpaths:
throw_away(fpath)
sigpath = fpath.parent / f'{fpath.name}.sig'
# there is a fscking problem that fscking pathlib always follow symlinks
if sigpath.exists() or sigpath.is_symlink():
throw_away(sigpath)
pkgnames = [
get_pkg_details_from_name(fpath.name).pkgname for fpath in fpaths
]
return bash(f'{REPO_REMOVE_CMD} {dbpath} {" ".join(pkgnames)}',
RUN_CMD_TIMEOUT=5 * 60)
def _update(overwrite=False):
logger.info('starting update')
update_path = Path('updates')
assert update_path.exists()
pkgs_to_add = dict()
filter_old_pkg(
[f for f in update_path.iterdir() if f.name.endswith(PKG_SUFFIX)],
keep_new=1,
archive=True)
for pkg_to_add in update_path.iterdir():
if pkg_to_add.is_dir():
continue
else:
if pkg_to_add.name.endswith(PKG_SUFFIX):
sigfile = Path(f"{pkg_to_add}.sig")
if sigfile.exists():
arch = get_pkg_details_from_name(pkg_to_add.name).arch
pkg_nlocation = pkg_to_add.parent / '..' / 'www' / arch / pkg_to_add.name
sig_nlocation = Path(f'{pkg_nlocation}.sig')
logger.info(
f'Copying {pkg_to_add} to {pkg_nlocation}, {sigfile} to {sig_nlocation}'
)
if overwrite:
for nlocation in (pkg_nlocation, sig_nlocation):
if nlocation.exists():
logger.warning(f'Overwriting {nlocation}')
else:
should_continue = False
for nlocation in (pkg_nlocation, sig_nlocation):
if nlocation.exists():
logger.warning(
'Same version is already in the repo.')
throw_away(pkg_to_add)
should_continue = True
break
if should_continue:
continue
copyfile(pkg_to_add, pkg_nlocation)
copyfile(sigfile, sig_nlocation)
archive_pkg(pkg_to_add)
archive_pkg(sigfile)
if arch == 'any':
for arch in ARCHS:
pkg_nlocation = pkg_to_add.parent / '..' / 'www' / arch / pkg_to_add.name
pkgs_to_add.setdefault(
arch, list()).append(pkg_nlocation)
else:
pkgs_to_add.setdefault(arch,
list()).append(pkg_nlocation)
else:
logger.warning(f'{pkg_to_add} has no signature!')
throw_away(pkg_to_add)
if 'any' in pkgs_to_add:
_regenerate(target_archs=ARCHS, just_symlink=True)
for arch in pkgs_to_add:
logger.info("repo-add: %s", repo_add(pkgs_to_add[arch]))
# remove add other things
for other in update_path.iterdir():
if other.is_dir():
continue
else:
logger.warning(f"{other} is garbage!")
throw_away(other)
logger.info('finished update')
return True
def _regenerate(target_archs=ARCHS, just_symlink=False):
if just_symlink:
logger.info('starting regenerate symlinks %s', target_archs)
else:
logger.info('starting regenerate %s', target_archs)
rn = REPO_NAME
repo_files = (f"{rn}.db {rn}.db.tar.gz {rn}.db.tar.gz.old "
f"{rn}.files {rn}.files.tar.gz {rn}.files.tar.gz.old")
repo_files = repo_files.split(' ')
repo_files_essential = [
fname for fname in repo_files if not fname.endswith('.old')
]
assert repo_files_essential
# make symlink for arch=any pkgs
basedir = Path('www') / 'any'
if basedir.exists():
for pkgfile in basedir.iterdir():
if pkgfile.name.endswith(PKG_SUFFIX) and \
get_pkg_details_from_name(pkgfile.name).arch == 'any':
sigfile = Path(f"{pkgfile}.sig")
if sigfile.exists():
logger.info(f'Creating symlink for {pkgfile}, {sigfile}')
for arch in target_archs:
if arch == 'any':
continue
symlink(pkgfile.parent / '..' / arch / pkgfile.name,
f'../any/{pkgfile.name}')
symlink(sigfile.parent / '..' / arch / sigfile.name,
f'../any/{sigfile.name}')
else:
logger.error(f'{arch} dir does not exist!')
if just_symlink:
return True
# run repo_add
for arch in target_archs:
basedir = Path('www') / arch
repo_files_count = list()
pkgs_to_add = list()
if not basedir.exists():
logger.error(f'{arch} dir does not exist!')
continue
filter_old_pkg(
[f for f in basedir.iterdir() if f.name.endswith(PKG_SUFFIX)],
keep_new=1,
recycle=True)
pkgfiles = [f for f in basedir.iterdir()]
for pkgfile in pkgfiles:
if pkgfile.name in repo_files:
repo_files_count.append(pkgfile.name)
continue
if pkgfile.name.endswith(PKG_SIG_SUFFIX):
if not Path(str(pkgfile)[:-4]).exists() and pkgfile.exists():
logger.warning(f"{pkgfile} has no package!")
throw_away(pkgfile)
continue
elif pkgfile.name.endswith(PKG_SUFFIX):
sigfile = Path(f"{pkgfile}.sig")
if not sigfile.exists():
logger.warning(f"{pkgfile} has no signature!")
throw_away(pkgfile)
continue
realarch = get_pkg_details_from_name(pkgfile.name).arch
if realarch != 'any' and realarch != arch:
newpath = pkgfile.parent / '..' / realarch / pkgfile.name
newSigpath = Path(f'{newpath}.sig')
logger.info(
f'Moving {pkgfile} to {newpath}, {sigfile} to {newSigpath}'
)
assert not (newpath.exists() or newSigpath.exists())
pkgfile.rename(newpath)
sigfile.rename(newSigpath)
pkgs_to_add.append(newpath)
else:
pkgs_to_add.append(pkgfile)
else:
logger.warning(f"{pkgfile} is garbage!")
throw_away(pkgfile)
if pkgs_to_add:
logger.info("repo-add: %s", repo_add(pkgs_to_add))
else:
logger.warning('repo-add: Nothing to do in %s', arch)
for rfile in repo_files_essential:
if rfile not in repo_files_count:
logger.error(f'{rfile} does not exist in {arch}!')
logger.info('finished regenerate')
return True
def __get_new_ver(self, dirname, arch):
pkgfiles = self.__get_package_list(dirname, arch)
ver = get_pkg_details_from_name(pkgfiles[0]).ver
return ver
def __upload(self, job, overwrite=False):
cwd = REPO_ROOT / job.pkgconfig.dirname
f_to_upload = list()
pkg_update_list = list()
for fpath in cwd.iterdir():
if fpath.name.endswith(PKG_SUFFIX) and \
get_pkg_details_from_name(fpath.name).ver == job.version:
sigpath = fpath.parent / f'{fpath.name}.sig'
assert sigpath.exists()
f_to_upload.append(sigpath)
f_to_upload.append(fpath)
pkg_update_list.append(fpath)
sizes = [f.stat().st_size / 1000 / 1000 for f in f_to_upload]
pkg_update_list_human = " ".join([f.name for f in pkg_update_list])
assert pkg_update_list
max_tries = 10
for tries in range(max_tries):
timeouts = rrun('push_start',
args=([f.name for f in f_to_upload], sizes))
if type(timeouts) is list:
break
else:
if tries + 1 < max_tries:
logger.warning(
f'Remote is busy ({timeouts}), wait 1 min x10 [{tries+1}/10]'
)
sleep(60)
else:
raise RuntimeError('Remote is busy and cannot connect')
assert len(f_to_upload) == len(timeouts)
pkgs_timeouts = {
f_to_upload[i]: timeouts[i]
for i in range(len(sizes))
}
for f in f_to_upload:
max_tries = 5
for tries in range(max_tries):
timeout = pkgs_timeouts.get(f)
try:
logger.info(f'Uploading {f.name}, timeout in {timeout}s')
mon_bash(UPLOAD_CMD.format(src=f), seconds=int(timeout))
except Exception:
time_to_sleep = (tries + 1) * 60
logger.error(
f'We are getting problem uploading {f.name}, wait {time_to_sleep} secs'
)
patret = rrun('push_add_time',
args=(f.name, time_to_sleep + timeout))
if not patret is None:
logger.error(
f'Unable to run push_add_time, reason: {patret}')
print_exc_plus()
if tries + 1 < max_tries:
sleep(time_to_sleep)
else:
break
else:
logger.error(
f'Upload {f.name} failed, running push_fail and abort.')
pfret = rrun('push_fail', args=(f.name, ))
if not pfret is None:
logger.error(f'Unable to run push_fail, reason: {pfret}')
raise RuntimeError('Unable to upload some files')
logger.info(f'Requesting repo update for {pkg_update_list_human}')
res = "unexpected"
max_tries = 5
for tries in range(max_tries):
try:
res = rrun('push_done',
args=([f.name for f in f_to_upload], ),
kwargs={
'overwrite': overwrite,
})
except Exception:
time_to_sleep = (tries + 1) * 60
logger.info(
f'Error updating {pkg_update_list_human}, wait {time_to_sleep} secs'
)
print_exc_plus()
if tries + 1 < max_tries:
sleep(time_to_sleep)
else:
break
else:
ret = f'Update failed for {pkg_update_list_human}: max reties exceeded'
logger.error(ret)
raise RuntimeError(ret)
if res is None:
logger.info(f'Update success for {pkg_update_list_human}')
else:
ret = f'Update failed for {pkg_update_list_human}, reason: {res}'
logger.error(ret)
raise RuntimeError(ret)
return res is None