예제 #1
0
파일: build_pkg.py 프로젝트: lots0logs/auta
def build_pkgs(last=False, pkg_info=None):
    if pkg_info is None:
        return False
    # Create our tmp directories
    result = os.path.join("/tmp", "result")
    cache = os.path.join("/var/tmp", "pkg_cache")
    for d in [result, cache]:
        if os.path.exists(d) and 'result' in d:
            shutil.rmtree(d)
            os.mkdir(d, 0o777)
        elif os.path.exists(d) and 'pkg_cache' in d:
            logger.info('@@-build_pkg.py-@@ 476 | Cleaning package cache....')
            db.set('building', 'Cleaning package cache.')
            for pcache in os.listdir(d):
                pcache = os.path.join(d, pcache)
                if not os.path.isdir(pcache):
                    logger.error('@@-build_pkg.py-@@ 479 | pcache is not a directory')
                    continue
                for pfile in os.listdir(pcache):
                    pname = re.search('^([a-z]|[0-9]|-|_)+(?=-\d|r|v)', pfile)
                    if not pname or pname == '':
                        continue
                    pname = pname.group(0)
                    pfile = os.path.join(pcache, pfile)
                    dtime = time.time()
                    if os.stat(pfile).st_mtime < (dtime - 7 * 86400) or status.all_packages().ismember(pname):
                        remove(pfile)
        else:
            os.mkdir(d, 0o777)
    dirs = ['/var/tmp/32build', '/var/tmp/32bit']
    for d in dirs:
        if os.path.exists(d):
            shutil.rmtree(d)
        os.mkdir(d, 0o777)
    # pkglist = db.lrange('queue', 0, -1)
    pkglist1 = ['1']
    in_dir_last = len([name for name in os.listdir(result)])
    db.set('pkg_count', in_dir_last)
    for i in range(len(pkglist1)):
        pkg = pkg_info.name
        if pkg and pkg is not None and pkg != '':
            pkgbuild_dir = pkg_info.build_path
            status.current_status = 'Building %s with makepkg' % pkg
            bld_obj = build_obj.get_build_object(pkg_obj=pkg_info)
            bld_obj.failed = False
            bld_obj.completed = False
            bld_obj.version_str = pkg_info.version_str
            bld_obj.start_str = datetime.datetime.now().strftime("%m/%d/%Y %I:%M%p")
            status.building_num = bld_obj.bnum
            status.building_start = bld_obj.start_str
            build_id = bld_obj.bnum
            tlmsg = 'Build <a href="/build/%s">%s</a> for <strong>%s</strong> started.' % (build_id, build_id, pkg)
            Timeline(msg=tlmsg, tl_type=3)
            pbuilds = pkg_info.builds()
            pbuilds.append(build_id)
            bld_obj.pkgname = pkg
            pkg_deps = pkg_info.depends() or []
            pkg_deps_str = ' '.join(pkg_deps)
            run_docker_clean(pkg)

            if pkg_info is not None and pkg_info.autosum == "True":
                build_env = ['_AUTOSUMS=True']
            else:
                build_env = ['_AUTOSUMS=False']
            if '/cinnamon/' in pkg_info.path:
                build_env.append('_ALEXPKG=True')
            else:
                build_env.append('_ALEXPKG=False')
            hconfig = docker_utils.create_pkgs_host_config(cache, pkgbuild_dir, result)
            try:
                container = doc.create_container("antergos/makepkg",
                                                 command="/makepkg/build.sh " + pkg_deps_str,
                                                 volumes=['/var/cache/pacman', '/makepkg', '/repo',
                                                          '/pkg', '/root/.gnupg', '/staging',
                                                          '/32bit', '/32build', '/result'],
                                                 environment=build_env, cpuset='0-3', name=pkg,
                                                 host_config=hconfig)
                if container.get('Warnings') and container.get('Warnings') != '':
                    logger.error(container.get('Warnings'))
            except Exception as err:
                logger.error('Create container failed. Error Msg: %s' % err)
                bld_obj.failed = True
                bld_obj.completed = False
                continue

            bld_obj.container = container.get('Id')

            try:
                doc.start(container.get('Id'))
                cont = bld_obj.container
                stream_process = Process(target=publish_build_ouput, args=(cont, bld_obj))
                stream_process.start()
                result = doc.wait(cont)
                if result is not 0:
                    bld_obj.failed = True
                    bld_obj.completed = False
                    logger.error('[CONTAINER EXIT CODE] Container %s exited. Return code was %s' % (pkg, result))
                else:
                    logger.info('[CONTAINER EXIT CODE] Container %s exited. Return code was %s' % (pkg, result))
                    bld_obj.failed = False
                    bld_obj.completed = True
            except Exception as err:
                logger.error('Start container failed. Error Msg: %s' % err)
                bld_obj.failed = True
                bld_obj.completed = False
                continue
            # db.publish('build-ouput', 'ENDOFLOG')
            # stream = doc.logs(container, stdout=True, stderr=True, timestamps=True)
            # log_stream = stream.split('\n')
            # db_filter_and_add(log_stream, this_log)

            # in_dir = len([name for name in os.listdir(result)])
            # last_count = int(db.get('pkg_count'))
            # logger.info('last count is %s %s' % (last_count, type(last_count)))
            # logger.info('in_dir is %s %s' % (in_dir, type(in_dir)))
            pkgs2sign = None
            if not bld_obj.failed:
                db.publish('build-output', 'Signing package..')
                pkgs2sign = glob.glob(
                    '/srv/antergos.info/repo/iso/testing/uefi/antergos-staging/x86_64/%s-***.xz' % pkg)
                pkgs2sign32 = glob.glob(
                    '/srv/antergos.info/repo/iso/testing/uefi/antergos-staging/i686/%s-***.xz' % pkg)
                pkgs2sign = pkgs2sign + pkgs2sign32
                logger.info('[PKGS TO SIGN] %s' % pkgs2sign)
                if pkgs2sign is not None and pkgs2sign != []:
                    try_sign = sign_pkgs.batch_sign(pkgs2sign)
                else:
                    try_sign = False
                if try_sign:
                    db.publish('build-output', 'Signature created successfully for %s' % pkg)
                    logger.info('[SIGN PKG] Signature created successfully for %s' % pkg)
                    db.publish('build-output', 'Updating staging repo database..')
                    update_main_repo(pkg, 'staging', bld_obj)
                else:
                    bld_obj.failed = True
                    bld_obj.completed = False

            if not bld_obj.failed:
                db.publish('build-output', 'Build completed successfully!')
                tlmsg = 'Build <a href="/build/%s">%s</a> for <strong>%s</strong> completed.' % (
                    build_id, build_id, pkg)
                Timeline(msg=tlmsg, tl_type=4)
                # db.incr('pkg_count', (in_dir - last_count))
                completed = status.completed()
                completed.rpush(build_id)
                bld_obj.review_stat = 'pending'
            else:
                tlmsg = 'Build <a href="/build/%s">%s</a> for <strong>%s</strong> failed.' % (build_id, build_id, pkg)
                Timeline(msg=tlmsg, tl_type=5)
                if pkgs2sign is not None:
                    for p in pkgs2sign:
                        remove(p)
                        remove(p + '.sig')
                failed = status.failed()
                failed.rpush(build_id)

            bld_obj.end_str = datetime.datetime.now().strftime("%m/%d/%Y %I:%M%p")

            status.container = ''
            status.building_num = ''
            status.building_start = ''

            if not bld_obj.failed:
                db.set('antbs:misc:cache_buster:flag', True)
                return True
            return False
예제 #2
0
파일: build_pkg.py 프로젝트: lots0logs/auta
def handle_hook(first=False, last=False):
    status.idle = False
    pull_from = 'antergos'
    packages = status.queue()

    if os.path.exists(REPO_DIR):
        remove(REPO_DIR)
    try:
        subprocess.check_call(
            ['git', 'clone', 'http://github.com/antergos/antergos-packages.git'],
            cwd='/opt')
        subprocess.check_call(['chmod', '-R', 'a+rw', REPO_DIR], cwd='/opt')
    except subprocess.CalledProcessError as err:
        logger.error(err)

    if status.iso_flag:
        status.iso_flag = False
        status.current_status = 'Building docker image.'
        status.iso_building = True
        image = docker_utils.maybe_build_mkarchiso()
        db.lrem('queue', 0, 'antergos-iso')
        db.lrem('queue', 0, 'antergos-iso.openbox')
        if image:
            archs = ['x86_64', 'i686']
            if db.get('isoMinimal') == 'True':
                iso_name = 'antergos-iso-minimal-'
            else:
                iso_name = 'antergos-iso-'
            for arch in archs:
                db.rpush('queue', iso_name + arch)
                version = datetime.datetime.now().strftime('%Y.%m.%d')
                pkgobj = package.get_pkg_object(iso_name + arch)
                pkgobj.save_to_db('version', version)
            build_iso()
        db.set('isoBuilding', 'False')
        db.set('isoMinimal', 'False')
        db.set('idle', "True")
        return True

    elif first and not status.iso_flag:
        status.current_status = 'Building docker image.'
        image = docker_utils.maybe_build_base_devel()
        if not image:
            return False

        logger.info('Checking database for packages.')
        status.current_status = 'Checking database for queued packages'

        all_deps = process_package_queue(packages)

        logger.info('All queued packages are in the database, checking deps to determine build order.')
        status.current_status = 'Determining build order by sorting package depends'
        if len(all_deps) > 1:
            topsort = check_deps(all_deps)
            check = []
            packages.delete()
            for p in topsort:
                # TODO: What if there is already a group of packages in queue prior to the current group?
                packages.append(p)

        logger.info('Check deps complete. Starting build_pkgs')
        logger.debug((packages, status.iso_flag))
        status.current_status = 'Check deps complete. Starting build container.'

    if not status.iso_flag and len(packages) > 0:
        pack = status.queue().lpop()
        if pack and pack is not None and pack != '':
            pkgobj = package.get_pkg_object(name=pack)
        else:
            return False

        rqjob = get_current_job(db)
        rqjob.meta['is_first'] = first
        rqjob.meta['is_last'] = last
        rqjob.meta['package'] = pkgobj.name
        rqjob.save()

        status.now_building = pkgobj.name
        built = build_pkgs(last, pkgobj)
        # TODO: Move this into its own method
        if built:
            completed = status.completed()
            failed = status.failed()
            blds = pkgobj.builds()
            total = len(blds)
            if total > 0:
                success = len([x for x in pkgobj.blds if x in completed])
                failure = len([x for x in pkgobj.blds if x in failed])
                if success > 0:
                    success = 100 * success / total
                else:
                    success = 0
                if failure > 0:
                    failure = 100 * failure / total
                else:
                    failure = 0
                pkgobj.success_rate = success
                pkgobj.failure_rate = failure
    if last:
        remove('/opt/antergos-packages')
        status.idle = True
        status.building = 'Idle'
        status.now_building = 'Idle'
        status.container = ''
        status.building_num = ''
        status.building_start = ''
        logger.info('All builds completed.')