def build_info(num):
"""
:param num:
:return:
"""
if not num:
abort(404)
try:
bld_obj = build_obj.get_build_object(bnum=num)
except Exception:
abort(404)
cont = status.container
log = bld_obj.log_str
if not log:
log = 'Unavailable'
log = log.decode("utf8")
if cont:
container = cont[:20]
else:
container = None
res = 'completed' if bld_obj.completed else 'failed'
return render_template("build_info.html", pkg=bld_obj.pkgname, ver=bld_obj.version_str, res=res,
start=bld_obj.start_str, end=bld_obj.end_str, bnum=bld_obj.bnum, container=container,
log=log, user=user)
def match_pkg_name_build_log(bnum=None, match=None):
if not bnum or not match:
return False
pname = build_obj.get_build_object(bnum=bnum)
logger.info(bnum)
if pname:
return match in pname
else:
return False
def build_pkg_now():
"""
:return:
"""
if request.method == 'POST':
pkgname = request.form['pkgname']
dev = request.form['dev']
if not pkgname or pkgname is None or pkgname == '':
abort(500)
pexists = status.all_packages
pexists = pexists.ismember(pkgname)
if not pexists:
try:
package.Package(name=pkgname)
if os.path.exists('/var/tmp/antergos-packages/' + pkgname) or 'antergos-iso' in pkgname:
pexists = True
except Exception:
pass
if pexists:
is_logged_in = user.is_authenticated()
p, a, rev_pending = get_build_info(1, 'completed', is_logged_in)
# logger.info(rev_pending)
pending = False
for bnum in rev_pending.keys():
bld_obj = build_obj.get_build_object(bnum=bnum)
if pkgname == bld_obj.pkgname:
pending = True
break
if pending:
flash('Unable to build %s because it is in "pending review" status.' % pkgname, category='error')
else:
if '-x86_64' in pkgname or '-i686' in pkgname:
if not status.iso_building:
status.iso_flag = True
if 'minimal' in pkgname:
status.iso_minimal = True
else:
status.iso_minimal = False
else:
logger.info('RATE LIMIT ON ANTERGOS ISO IN EFFECT')
return redirect(redirect_url())
q = status.hook_queue
q.rpush(pkgname)
hook_queue.enqueue_call(builder.handle_hook, timeout=84600)
tl_event(
msg='<strong>%s</strong> added <strong>%s</strong> to the build queue.' % (dev, pkgname),
tl_type='0')
else:
flash('Package not found. Has the PKGBUILD been pushed to github?', category='error')
return redirect(redirect_url())
def set_pkg_review_result(bnum=None, dev=None, result=None):
# TODO: This is garbage. Needs rewrite.
if any(i is None for i in (bnum, dev, result)):
abort(500)
errmsg = dict(error=True, msg=None)
dt = datetime.now().strftime("%m/%d/%Y %I:%M%p")
try:
bld_obj = build_obj.get_build_object(bnum=bnum)
status.idle = False
pkg_obj = package.Package(name=bld_obj.pkgname)
if pkg_obj:
if 'main' not in pkg_obj.allowed_in and result == 'passed':
msg = '%s is not allowed in main repo.' % pkg_obj.pkgname
errmsg.update(error=True, msg=msg)
return errmsg
else:
bld_obj.review_dev = dev
bld_obj.review_date = dt
bld_obj.review_status = result
pkg_files_64 = glob.glob('%s/%s-***' % (STAGING_64, pkg_obj.pkgname))
pkg_files_32 = glob.glob('%s/%s-***' % (STAGING_32, pkg_obj.pkgname))
pkg_files = pkg_files_64 + pkg_files_32
if result == 'skip':
return errmsg
if pkg_files:
logger.info('Moving %s from staging to main repo.', pkg_obj.pkgname)
for f in pkg_files_64:
if result == 'passed':
copy(f, MAIN_64)
copy(f, '/tmp')
elif result == 'failed':
os.remove(f)
for f in pkg_files_32:
if result == 'passed':
copy(f, MAIN_32)
copy(f, '/tmp')
elif result == 'failed':
os.remove(f)
if result and result != 'skip':
repo_queue.enqueue_call(builder.update_main_repo, args=(pkg_obj.pkgname, result), timeout=9600)
else:
logger.error('While moving to main, no packages were found to move.')
err = 'While moving to main, no packages were found to move.'
errmsg = dict(error=True, msg=err)
except (OSError, Exception) as err:
logger.error('@@-antbs.py-@@ | Error while moving to main: %s', err)
err = str(err)
errmsg = dict(error=True, msg=err)
return errmsg
def homepage(tlpage=None):
"""
:param tlpage:
:return:
"""
if tlpage is None:
tlpage = 1
check_stats = ['queue', 'completed', 'failed']
building = status.current_status
this_page, all_pages = get_timeline(tlpage)
stats = {}
for stat in check_stats:
builds = getattr(status, stat)
res = len(builds)
if stat != "queue":
builds = [x for x in builds if x is not None]
within = []
nodup = []
for bnum in builds:
try:
bld_obj = build_obj.get_build_object(bnum=bnum)
except (ValueError, AttributeError):
continue
ver = '%s:%s' % (bld_obj.pkgname, bld_obj.version_str)
end = datetime.strptime(bld_obj.end_str, '%m/%d/%Y %I:%M%p') if bld_obj.end_str != '' else ''
if end != '' and (datetime.now() - end) < timedelta(hours=48) and ver not in nodup and bld_obj.pkgname:
within.append(bld_obj.bnum)
nodup.append(ver)
stats[stat] = len(within)
else:
stats[stat] = res
main_repo = glob.glob('/srv/antergos.info/repo/antergos/x86_64/*.pkg.tar.xz')
staging_repo = glob.glob('/srv/antergos.info/repo/iso/testing/uefi/antergos-staging/x86_64/*.pkg.tar.xz')
for repo in [main_repo, staging_repo]:
filtered = []
for file_path in repo:
new_fp = os.path.basename(file_path)
if 'dummy-package' not in new_fp:
filtered.append(new_fp)
if '-staging' not in repo[0]:
repo_name = 'repo_main'
else:
repo_name = 'repo_staging'
stats[repo_name] = len(set(filtered))
return render_template("overview.html", stats=stats, user=user, building=building,
this_page=this_page, all_pages=all_pages, page=tlpage, rev_pending=[])
def get_build_info(page=None, build_status=None, logged_in=False, search=None):
"""
:param page:
:param build_status:
:param logged_in:
:param search:
:return:
"""
if page is None or build_status is None:
abort(500)
if 'antergos' in build_status:
build_status = 'completed'
pkg_list = {}
rev_pending = {}
all_builds = None
all_pages = 0
try:
all_builds = getattr(status, build_status)
except Exception as err:
logger.error('GET_BUILD_INFO - %s', err)
abort(500)
if all_builds:
if search is not None:
search_all_builds = [x for x in all_builds if x is not None and match_pkg_name_build_log(x, search)]
logger.info('search_all_builds is %s', search_all_builds)
all_builds = search_all_builds
if all_builds:
builds, all_pages = get_paginated(all_builds, 10, page, False)
for bnum in builds:
try:
bld_obj = build_obj.get_build_object(bnum=bnum)
except Exception as err:
logger.error('Unable to ge build object - %s' % err)
continue
all_info = dict(bnum=bld_obj.bnum, name=bld_obj.pkgname, version=bld_obj.version_str,
start=bld_obj.start_str, end=bld_obj.end_str, review_stat=bld_obj.review_status,
review_dev=bld_obj.review_dev, review_date=bld_obj.review_date)
pkg_list[bnum] = all_info
if logged_in and bld_obj.review_status == "pending":
rev_pending[bnum] = all_info
return pkg_list, int(all_pages), rev_pending
def build():
now_building = status.now_building
cont = status.container
if cont:
container = cont[:20]
else:
container = None
bnum = status.building_num
start = status.building_start
try:
bld_obj = build_obj.get_build_object(bnum=bnum)
ver = bld_obj.version_str
except Exception as err:
logger.error(err)
ver = ''
return render_template("building.html", building=now_building, container=container, bnum=bnum,
start=start, ver=ver)
def get_repo_info(repo=None, logged_in=False):
"""
:param repo:
:param logged_in:
:return:
"""
if repo is None:
abort(500)
pkg_list = {}
p, a, rev_pending = get_build_info(1, repo, logged_in)
logger.info('@@-antbs.py-@@ 295 | GET_REPO_INFO - CACHE CHECK FAILED. WE ARE NOT USING CACHED INFO')
all_packages = glob.glob('/srv/antergos.info/repo/%s/x86_64/***.pkg.tar.xz' % repo)
if all_packages is not None:
for item in all_packages:
# logger.info(item)
item = item.split('/')[-1]
item = re.search('^([a-z]|[0-9]|-|_)+(?=-\d|r|v)', item)
item = item.group(0) or ''
if not item or item == '':
continue
logger.info(item)
pkg = package.get_pkg_object(item)
builds = pkg.builds
try:
bnum = builds[0]
except Exception:
bnum = ''
bld_obj = build_obj.get_build_object(bnum=bnum)
all_info = dict(bnum=bnum, name=pkg.name, version=pkg.version_str, review_dev=bld_obj.review_dev,
review_stat=bld_obj.review_stat, review_date=bld_obj.review_date, pkgid=pkg.pkgid)
pkg_list[pkg.pkgid] = all_info
return pkg_list, rev_pending
def process_and_save_build_metadata(pkg_obj=None):
"""
:param pkg_obj:
:return: :raise AttributeError:
"""
if not pkg_obj:
raise AttributeError
status.current_status = 'Building %s' % pkg_obj.name
status.now_building = pkg_obj.name
logger.info('Building %s' % pkg_obj.name)
bld_obj = build_obj.get_build_object(pkg_obj=pkg_obj)
bld_obj.start_str = datetime.datetime.now().strftime("%m/%d/%Y %I:%M%p")
status.building_num = bld_obj.bnum
status.building_start = bld_obj.start_str
build_id = bld_obj.bnum
tlmsg = 'Build <a href="/build/%s">%s</a> for <strong>%s</strong> started.' % (build_id, build_id, pkg_obj.name)
Timeline(msg=tlmsg, tl_type=3)
pbuilds = pkg_obj.builds
pbuilds.append(build_id)
run_docker_clean(pkg_obj.name)
return bld_obj
def build():
"""
:return:
"""
now_building = status.now_building
ver = ''
bnum = ''
start = ''
cont = status.container
if cont:
container = cont[:20]
else:
container = None
if not status.idle:
bnum = status.building_num
start = status.building_start
if bnum and bnum != '':
bld_obj = build_obj.get_build_object(bnum=bnum)
ver = bld_obj.version_str
return render_template("building.html", building=now_building, container=container, bnum=bnum,
start=start, ver=ver)
def build_pkgs(last=False, pkg_info=None):
if pkg_info is None:
return False
# Create our tmp directories
result = os.path.join("/tmp", "result")
cache = os.path.join("/var/tmp", "pkg_cache")
for d in [result, cache]:
if os.path.exists(d) and 'result' in d:
shutil.rmtree(d)
os.mkdir(d, 0o777)
elif os.path.exists(d) and 'pkg_cache' in d:
logger.info('@@-build_pkg.py-@@ 476 | Cleaning package cache....')
db.set('building', 'Cleaning package cache.')
for pcache in os.listdir(d):
pcache = os.path.join(d, pcache)
if not os.path.isdir(pcache):
logger.error('@@-build_pkg.py-@@ 479 | pcache is not a directory')
continue
for pfile in os.listdir(pcache):
pname = re.search('^([a-z]|[0-9]|-|_)+(?=-\d|r|v)', pfile)
if not pname or pname == '':
continue
pname = pname.group(0)
pfile = os.path.join(pcache, pfile)
dtime = time.time()
if os.stat(pfile).st_mtime < (dtime - 7 * 86400) or status.all_packages().ismember(pname):
remove(pfile)
else:
os.mkdir(d, 0o777)
dirs = ['/var/tmp/32build', '/var/tmp/32bit']
for d in dirs:
if os.path.exists(d):
shutil.rmtree(d)
os.mkdir(d, 0o777)
# pkglist = db.lrange('queue', 0, -1)
pkglist1 = ['1']
in_dir_last = len([name for name in os.listdir(result)])
db.set('pkg_count', in_dir_last)
for i in range(len(pkglist1)):
pkg = pkg_info.name
if pkg and pkg is not None and pkg != '':
pkgbuild_dir = pkg_info.build_path
status.current_status = 'Building %s with makepkg' % pkg
bld_obj = build_obj.get_build_object(pkg_obj=pkg_info)
bld_obj.failed = False
bld_obj.completed = False
bld_obj.version_str = pkg_info.version_str
bld_obj.start_str = datetime.datetime.now().strftime("%m/%d/%Y %I:%M%p")
status.building_num = bld_obj.bnum
status.building_start = bld_obj.start_str
build_id = bld_obj.bnum
tlmsg = 'Build <a href="/build/%s">%s</a> for <strong>%s</strong> started.' % (build_id, build_id, pkg)
Timeline(msg=tlmsg, tl_type=3)
pbuilds = pkg_info.builds()
pbuilds.append(build_id)
bld_obj.pkgname = pkg
pkg_deps = pkg_info.depends() or []
pkg_deps_str = ' '.join(pkg_deps)
run_docker_clean(pkg)
if pkg_info is not None and pkg_info.autosum == "True":
build_env = ['_AUTOSUMS=True']
else:
build_env = ['_AUTOSUMS=False']
if '/cinnamon/' in pkg_info.path:
build_env.append('_ALEXPKG=True')
else:
build_env.append('_ALEXPKG=False')
hconfig = docker_utils.create_pkgs_host_config(cache, pkgbuild_dir, result)
try:
container = doc.create_container("antergos/makepkg",
command="/makepkg/build.sh " + pkg_deps_str,
volumes=['/var/cache/pacman', '/makepkg', '/repo',
'/pkg', '/root/.gnupg', '/staging',
'/32bit', '/32build', '/result'],
environment=build_env, cpuset='0-3', name=pkg,
host_config=hconfig)
if container.get('Warnings') and container.get('Warnings') != '':
logger.error(container.get('Warnings'))
except Exception as err:
logger.error('Create container failed. Error Msg: %s' % err)
bld_obj.failed = True
bld_obj.completed = False
continue
bld_obj.container = container.get('Id')
try:
doc.start(container.get('Id'))
cont = bld_obj.container
stream_process = Process(target=publish_build_ouput, args=(cont, bld_obj))
stream_process.start()
result = doc.wait(cont)
if result is not 0:
bld_obj.failed = True
bld_obj.completed = False
logger.error('[CONTAINER EXIT CODE] Container %s exited. Return code was %s' % (pkg, result))
else:
logger.info('[CONTAINER EXIT CODE] Container %s exited. Return code was %s' % (pkg, result))
bld_obj.failed = False
bld_obj.completed = True
except Exception as err:
logger.error('Start container failed. Error Msg: %s' % err)
bld_obj.failed = True
bld_obj.completed = False
continue
# db.publish('build-ouput', 'ENDOFLOG')
# stream = doc.logs(container, stdout=True, stderr=True, timestamps=True)
# log_stream = stream.split('\n')
# db_filter_and_add(log_stream, this_log)
# in_dir = len([name for name in os.listdir(result)])
# last_count = int(db.get('pkg_count'))
# logger.info('last count is %s %s' % (last_count, type(last_count)))
# logger.info('in_dir is %s %s' % (in_dir, type(in_dir)))
pkgs2sign = None
if not bld_obj.failed:
db.publish('build-output', 'Signing package..')
pkgs2sign = glob.glob(
'/srv/antergos.info/repo/iso/testing/uefi/antergos-staging/x86_64/%s-***.xz' % pkg)
pkgs2sign32 = glob.glob(
'/srv/antergos.info/repo/iso/testing/uefi/antergos-staging/i686/%s-***.xz' % pkg)
pkgs2sign = pkgs2sign + pkgs2sign32
logger.info('[PKGS TO SIGN] %s' % pkgs2sign)
if pkgs2sign is not None and pkgs2sign != []:
try_sign = sign_pkgs.batch_sign(pkgs2sign)
else:
try_sign = False
if try_sign:
db.publish('build-output', 'Signature created successfully for %s' % pkg)
logger.info('[SIGN PKG] Signature created successfully for %s' % pkg)
db.publish('build-output', 'Updating staging repo database..')
update_main_repo(pkg, 'staging', bld_obj)
else:
bld_obj.failed = True
bld_obj.completed = False
if not bld_obj.failed:
db.publish('build-output', 'Build completed successfully!')
tlmsg = 'Build <a href="/build/%s">%s</a> for <strong>%s</strong> completed.' % (
build_id, build_id, pkg)
Timeline(msg=tlmsg, tl_type=4)
# db.incr('pkg_count', (in_dir - last_count))
completed = status.completed()
completed.rpush(build_id)
bld_obj.review_stat = 'pending'
else:
tlmsg = 'Build <a href="/build/%s">%s</a> for <strong>%s</strong> failed.' % (build_id, build_id, pkg)
Timeline(msg=tlmsg, tl_type=5)
if pkgs2sign is not None:
for p in pkgs2sign:
remove(p)
remove(p + '.sig')
failed = status.failed()
failed.rpush(build_id)
bld_obj.end_str = datetime.datetime.now().strftime("%m/%d/%Y %I:%M%p")
status.container = ''
status.building_num = ''
status.building_start = ''
if not bld_obj.failed:
db.set('antbs:misc:cache_buster:flag', True)
return True
return False
def set_pkg_review_result(bnum=None, dev=None, result=None):
# TODO: This is garbage. Needs rewrite.
"""
:param bnum:
:param dev:
:param result:
:return:
"""
if any(i is None for i in (bnum, dev, result)):
abort(500)
errmsg = dict(error=True, msg=None)
dt = datetime.now().strftime("%m/%d/%Y %I:%M%p")
if result in ['0', '1', '2', '3', '4']:
msg = 'Please clear your browser cache, refresh the page, and try again.'
errmsg.update(error=True, msg=msg)
return errmsg
try:
bld_obj = build_obj.get_build_object(bnum=bnum)
pkg_obj = package.Package(name=bld_obj.pkgname)
if pkg_obj and build_obj:
allowed = pkg_obj.allowed_in
if 'main' not in allowed and result == 'passed':
msg = '%s is not allowed in main repo.' % pkg_obj.pkgname
errmsg.update(error=True, msg=msg)
return errmsg
else:
bld_obj.review_dev = dev
bld_obj.review_date = dt
bld_obj.review_status = result
if result == 'skip':
errmsg = dict(error=False, msg=None)
return errmsg
pkg_files_64 = glob.glob('%s/%s-***' % (STAGING_64, pkg_obj.pkgname))
pkg_files_32 = glob.glob('%s/%s-***' % (STAGING_32, pkg_obj.pkgname))
pkg_files = pkg_files_64 + pkg_files_32
if pkg_files or True:
for f in pkg_files_64:
logger.debug('f in pkg_files_64 fired!')
if result == 'passed':
copy(f, MAIN_64)
copy(f, '/tmp')
elif result == 'failed':
os.remove(f)
for f in pkg_files_32:
if result == 'passed':
copy(f, MAIN_32)
copy(f, '/tmp')
elif result == 'failed':
os.remove(f)
if result and result != 'skip':
repo_queue.enqueue_call(builder.update_main_repo, (result, None, True, bld_obj.pkgname), timeout=9600)
errmsg = dict(error=False, msg=None)
else:
logger.error('While moving to main, no packages were found to move.')
err = 'While moving to main, no packages were found to move.'
errmsg = dict(error=True, msg=err)
except (OSError, Exception) as err:
logger.error('@@-antbs.py-@@ | Error while moving to main: %s', err)
err = str(err)
errmsg = dict(error=True, msg=err)
return errmsg