def build_pkgs(last=False, pkg_info=None):
if pkg_info is None:
return False
# Create our tmp directories
result = os.path.join("/tmp", "result")
cache = os.path.join("/var/tmp", "pkg_cache")
for d in [result, cache]:
if os.path.exists(d) and 'result' in d:
shutil.rmtree(d)
os.mkdir(d, 0o777)
elif os.path.exists(d) and 'pkg_cache' in d:
logger.info('@@-build_pkg.py-@@ 476 | Cleaning package cache....')
db.set('building', 'Cleaning package cache.')
for pcache in os.listdir(d):
pcache = os.path.join(d, pcache)
if not os.path.isdir(pcache):
logger.error('@@-build_pkg.py-@@ 479 | pcache is not a directory')
continue
for pfile in os.listdir(pcache):
pname = re.search('^([a-z]|[0-9]|-|_)+(?=-\d|r|v)', pfile)
if not pname or pname == '':
continue
pname = pname.group(0)
pfile = os.path.join(pcache, pfile)
dtime = time.time()
if os.stat(pfile).st_mtime < (dtime - 7 * 86400) or status.all_packages().ismember(pname):
remove(pfile)
else:
os.mkdir(d, 0o777)
dirs = ['/var/tmp/32build', '/var/tmp/32bit']
for d in dirs:
if os.path.exists(d):
shutil.rmtree(d)
os.mkdir(d, 0o777)
# pkglist = db.lrange('queue', 0, -1)
pkglist1 = ['1']
in_dir_last = len([name for name in os.listdir(result)])
db.set('pkg_count', in_dir_last)
for i in range(len(pkglist1)):
pkg = pkg_info.name
if pkg and pkg is not None and pkg != '':
pkgbuild_dir = pkg_info.build_path
status.current_status = 'Building %s with makepkg' % pkg
bld_obj = build_obj.get_build_object(pkg_obj=pkg_info)
bld_obj.failed = False
bld_obj.completed = False
bld_obj.version_str = pkg_info.version_str
bld_obj.start_str = datetime.datetime.now().strftime("%m/%d/%Y %I:%M%p")
status.building_num = bld_obj.bnum
status.building_start = bld_obj.start_str
build_id = bld_obj.bnum
tlmsg = 'Build <a href="/build/%s">%s</a> for <strong>%s</strong> started.' % (build_id, build_id, pkg)
Timeline(msg=tlmsg, tl_type=3)
pbuilds = pkg_info.builds()
pbuilds.append(build_id)
bld_obj.pkgname = pkg
pkg_deps = pkg_info.depends() or []
pkg_deps_str = ' '.join(pkg_deps)
run_docker_clean(pkg)
if pkg_info is not None and pkg_info.autosum == "True":
build_env = ['_AUTOSUMS=True']
else:
build_env = ['_AUTOSUMS=False']
if '/cinnamon/' in pkg_info.path:
build_env.append('_ALEXPKG=True')
else:
build_env.append('_ALEXPKG=False')
hconfig = docker_utils.create_pkgs_host_config(cache, pkgbuild_dir, result)
try:
container = doc.create_container("antergos/makepkg",
command="/makepkg/build.sh " + pkg_deps_str,
volumes=['/var/cache/pacman', '/makepkg', '/repo',
'/pkg', '/root/.gnupg', '/staging',
'/32bit', '/32build', '/result'],
environment=build_env, cpuset='0-3', name=pkg,
host_config=hconfig)
if container.get('Warnings') and container.get('Warnings') != '':
logger.error(container.get('Warnings'))
except Exception as err:
logger.error('Create container failed. Error Msg: %s' % err)
bld_obj.failed = True
bld_obj.completed = False
continue
bld_obj.container = container.get('Id')
try:
doc.start(container.get('Id'))
cont = bld_obj.container
stream_process = Process(target=publish_build_ouput, args=(cont, bld_obj))
stream_process.start()
result = doc.wait(cont)
if result is not 0:
bld_obj.failed = True
bld_obj.completed = False
logger.error('[CONTAINER EXIT CODE] Container %s exited. Return code was %s' % (pkg, result))
else:
logger.info('[CONTAINER EXIT CODE] Container %s exited. Return code was %s' % (pkg, result))
bld_obj.failed = False
bld_obj.completed = True
except Exception as err:
logger.error('Start container failed. Error Msg: %s' % err)
bld_obj.failed = True
bld_obj.completed = False
continue
# db.publish('build-ouput', 'ENDOFLOG')
# stream = doc.logs(container, stdout=True, stderr=True, timestamps=True)
# log_stream = stream.split('\n')
# db_filter_and_add(log_stream, this_log)
# in_dir = len([name for name in os.listdir(result)])
# last_count = int(db.get('pkg_count'))
# logger.info('last count is %s %s' % (last_count, type(last_count)))
# logger.info('in_dir is %s %s' % (in_dir, type(in_dir)))
pkgs2sign = None
if not bld_obj.failed:
db.publish('build-output', 'Signing package..')
pkgs2sign = glob.glob(
'/srv/antergos.info/repo/iso/testing/uefi/antergos-staging/x86_64/%s-***.xz' % pkg)
pkgs2sign32 = glob.glob(
'/srv/antergos.info/repo/iso/testing/uefi/antergos-staging/i686/%s-***.xz' % pkg)
pkgs2sign = pkgs2sign + pkgs2sign32
logger.info('[PKGS TO SIGN] %s' % pkgs2sign)
if pkgs2sign is not None and pkgs2sign != []:
try_sign = sign_pkgs.batch_sign(pkgs2sign)
else:
try_sign = False
if try_sign:
db.publish('build-output', 'Signature created successfully for %s' % pkg)
logger.info('[SIGN PKG] Signature created successfully for %s' % pkg)
db.publish('build-output', 'Updating staging repo database..')
update_main_repo(pkg, 'staging', bld_obj)
else:
bld_obj.failed = True
bld_obj.completed = False
if not bld_obj.failed:
db.publish('build-output', 'Build completed successfully!')
tlmsg = 'Build <a href="/build/%s">%s</a> for <strong>%s</strong> completed.' % (
build_id, build_id, pkg)
Timeline(msg=tlmsg, tl_type=4)
# db.incr('pkg_count', (in_dir - last_count))
completed = status.completed()
completed.rpush(build_id)
bld_obj.review_stat = 'pending'
else:
tlmsg = 'Build <a href="/build/%s">%s</a> for <strong>%s</strong> failed.' % (build_id, build_id, pkg)
Timeline(msg=tlmsg, tl_type=5)
if pkgs2sign is not None:
for p in pkgs2sign:
remove(p)
remove(p + '.sig')
failed = status.failed()
failed.rpush(build_id)
bld_obj.end_str = datetime.datetime.now().strftime("%m/%d/%Y %I:%M%p")
status.container = ''
status.building_num = ''
status.building_start = ''
if not bld_obj.failed:
db.set('antbs:misc:cache_buster:flag', True)
return True
return False
def handle_hook(first=False, last=False):
status.idle = False
pull_from = 'antergos'
packages = status.queue()
if os.path.exists(REPO_DIR):
remove(REPO_DIR)
try:
subprocess.check_call(
['git', 'clone', 'http://github.com/antergos/antergos-packages.git'],
cwd='/opt')
subprocess.check_call(['chmod', '-R', 'a+rw', REPO_DIR], cwd='/opt')
except subprocess.CalledProcessError as err:
logger.error(err)
if status.iso_flag:
status.iso_flag = False
status.current_status = 'Building docker image.'
status.iso_building = True
image = docker_utils.maybe_build_mkarchiso()
db.lrem('queue', 0, 'antergos-iso')
db.lrem('queue', 0, 'antergos-iso.openbox')
if image:
archs = ['x86_64', 'i686']
if db.get('isoMinimal') == 'True':
iso_name = 'antergos-iso-minimal-'
else:
iso_name = 'antergos-iso-'
for arch in archs:
db.rpush('queue', iso_name + arch)
version = datetime.datetime.now().strftime('%Y.%m.%d')
pkgobj = package.get_pkg_object(iso_name + arch)
pkgobj.save_to_db('version', version)
build_iso()
db.set('isoBuilding', 'False')
db.set('isoMinimal', 'False')
db.set('idle', "True")
return True
elif first and not status.iso_flag:
status.current_status = 'Building docker image.'
image = docker_utils.maybe_build_base_devel()
if not image:
return False
logger.info('Checking database for packages.')
status.current_status = 'Checking database for queued packages'
all_deps = process_package_queue(packages)
logger.info('All queued packages are in the database, checking deps to determine build order.')
status.current_status = 'Determining build order by sorting package depends'
if len(all_deps) > 1:
topsort = check_deps(all_deps)
check = []
packages.delete()
for p in topsort:
# TODO: What if there is already a group of packages in queue prior to the current group?
packages.append(p)
logger.info('Check deps complete. Starting build_pkgs')
logger.debug((packages, status.iso_flag))
status.current_status = 'Check deps complete. Starting build container.'
if not status.iso_flag and len(packages) > 0:
pack = status.queue().lpop()
if pack and pack is not None and pack != '':
pkgobj = package.get_pkg_object(name=pack)
else:
return False
rqjob = get_current_job(db)
rqjob.meta['is_first'] = first
rqjob.meta['is_last'] = last
rqjob.meta['package'] = pkgobj.name
rqjob.save()
status.now_building = pkgobj.name
built = build_pkgs(last, pkgobj)
# TODO: Move this into its own method
if built:
completed = status.completed()
failed = status.failed()
blds = pkgobj.builds()
total = len(blds)
if total > 0:
success = len([x for x in pkgobj.blds if x in completed])
failure = len([x for x in pkgobj.blds if x in failed])
if success > 0:
success = 100 * success / total
else:
success = 0
if failure > 0:
failure = 100 * failure / total
else:
failure = 0
pkgobj.success_rate = success
pkgobj.failure_rate = failure
if last:
remove('/opt/antergos-packages')
status.idle = True
status.building = 'Idle'
status.now_building = 'Idle'
status.container = ''
status.building_num = ''
status.building_start = ''
logger.info('All builds completed.')