Esempio n. 1
0
def build_info(num):
    """

    :param num:
    :return:
    """
    if not num:
        abort(404)
    try:
        bld_obj = build_obj.get_build_object(bnum=num)
    except Exception:
        abort(404)

    cont = status.container
    log = bld_obj.log_str
    if not log:
        log = 'Unavailable'
    log = log.decode("utf8")
    if cont:
        container = cont[:20]
    else:
        container = None
    res = 'completed' if bld_obj.completed else 'failed'

    return render_template("build_info.html", pkg=bld_obj.pkgname, ver=bld_obj.version_str, res=res,
                           start=bld_obj.start_str, end=bld_obj.end_str, bnum=bld_obj.bnum, container=container,
                           log=log, user=user)
Esempio n. 2
0
def match_pkg_name_build_log(bnum=None, match=None):
    if not bnum or not match:
        return False
    pname = build_obj.get_build_object(bnum=bnum)
    logger.info(bnum)
    if pname:
        return match in pname
    else:
        return False
Esempio n. 3
0
def build_pkg_now():
    """


    :return:
    """
    if request.method == 'POST':
        pkgname = request.form['pkgname']
        dev = request.form['dev']
        if not pkgname or pkgname is None or pkgname == '':
            abort(500)
        pexists = status.all_packages
        pexists = pexists.ismember(pkgname)
        if not pexists:
            try:
                package.Package(name=pkgname)
                if os.path.exists('/var/tmp/antergos-packages/' + pkgname) or 'antergos-iso' in pkgname:
                    pexists = True
            except Exception:
                pass

        if pexists:
            is_logged_in = user.is_authenticated()
            p, a, rev_pending = get_build_info(1, 'completed', is_logged_in)
            # logger.info(rev_pending)
            pending = False
            for bnum in rev_pending.keys():
                bld_obj = build_obj.get_build_object(bnum=bnum)
                if pkgname == bld_obj.pkgname:
                    pending = True
                    break

            if pending:
                flash('Unable to build %s because it is in "pending review" status.' % pkgname, category='error')
            else:
                if '-x86_64' in pkgname or '-i686' in pkgname:
                    if not status.iso_building:
                        status.iso_flag = True
                        if 'minimal' in pkgname:
                            status.iso_minimal = True
                        else:
                            status.iso_minimal = False
                    else:
                        logger.info('RATE LIMIT ON ANTERGOS ISO IN EFFECT')
                        return redirect(redirect_url())

                q = status.hook_queue
                q.rpush(pkgname)
                hook_queue.enqueue_call(builder.handle_hook, timeout=84600)
                tl_event(
                    msg='<strong>%s</strong> added <strong>%s</strong> to the build queue.' % (dev, pkgname),
                    tl_type='0')
        else:
            flash('Package not found. Has the PKGBUILD been pushed to github?', category='error')

    return redirect(redirect_url())
Esempio n. 4
0
def set_pkg_review_result(bnum=None, dev=None, result=None):
    # TODO: This is garbage. Needs rewrite.
    if any(i is None for i in (bnum, dev, result)):
        abort(500)
    errmsg = dict(error=True, msg=None)
    dt = datetime.now().strftime("%m/%d/%Y %I:%M%p")
    try:
        bld_obj = build_obj.get_build_object(bnum=bnum)
        status.idle = False
        pkg_obj = package.Package(name=bld_obj.pkgname)
        if pkg_obj:
            if 'main' not in pkg_obj.allowed_in and result == 'passed':
                msg = '%s is not allowed in main repo.' % pkg_obj.pkgname
                errmsg.update(error=True, msg=msg)
                return errmsg
            else:
                bld_obj.review_dev = dev
                bld_obj.review_date = dt
                bld_obj.review_status = result

        pkg_files_64 = glob.glob('%s/%s-***' % (STAGING_64, pkg_obj.pkgname))
        pkg_files_32 = glob.glob('%s/%s-***' % (STAGING_32, pkg_obj.pkgname))
        pkg_files = pkg_files_64 + pkg_files_32

        if result == 'skip':
            return errmsg
        if pkg_files:
            logger.info('Moving %s from staging to main repo.', pkg_obj.pkgname)
            for f in pkg_files_64:
                if result == 'passed':
                    copy(f, MAIN_64)
                    copy(f, '/tmp')
                elif result == 'failed':
                    os.remove(f)
            for f in pkg_files_32:
                if result == 'passed':
                    copy(f, MAIN_32)
                    copy(f, '/tmp')
                elif result == 'failed':
                    os.remove(f)
            if result and result != 'skip':
                repo_queue.enqueue_call(builder.update_main_repo, args=(pkg_obj.pkgname, result), timeout=9600)

        else:
            logger.error('While moving to main, no packages were found to move.')
            err = 'While moving to main, no packages were found to move.'
            errmsg = dict(error=True, msg=err)

    except (OSError, Exception) as err:
        logger.error('@@-antbs.py-@@ | Error while moving to main: %s', err)
        err = str(err)
        errmsg = dict(error=True, msg=err)

    return errmsg
Esempio n. 5
0
def homepage(tlpage=None):
    """

    :param tlpage:
    :return:
    """
    if tlpage is None:
        tlpage = 1
    check_stats = ['queue', 'completed', 'failed']
    building = status.current_status
    this_page, all_pages = get_timeline(tlpage)

    stats = {}
    for stat in check_stats:
        builds = getattr(status, stat)
        res = len(builds)
        if stat != "queue":
            builds = [x for x in builds if x is not None]
            within = []
            nodup = []
            for bnum in builds:
                try:
                    bld_obj = build_obj.get_build_object(bnum=bnum)
                except (ValueError, AttributeError):
                    continue
                ver = '%s:%s' % (bld_obj.pkgname, bld_obj.version_str)
                end = datetime.strptime(bld_obj.end_str, '%m/%d/%Y %I:%M%p') if bld_obj.end_str != '' else ''
                if end != '' and (datetime.now() - end) < timedelta(hours=48) and ver not in nodup and bld_obj.pkgname:
                    within.append(bld_obj.bnum)
                    nodup.append(ver)

            stats[stat] = len(within)
        else:
            stats[stat] = res

    main_repo = glob.glob('/srv/antergos.info/repo/antergos/x86_64/*.pkg.tar.xz')
    staging_repo = glob.glob('/srv/antergos.info/repo/iso/testing/uefi/antergos-staging/x86_64/*.pkg.tar.xz')

    for repo in [main_repo, staging_repo]:
        filtered = []
        for file_path in repo:
            new_fp = os.path.basename(file_path)
            if 'dummy-package' not in new_fp:
                filtered.append(new_fp)
        if '-staging' not in repo[0]:
            repo_name = 'repo_main'
        else:
            repo_name = 'repo_staging'
        stats[repo_name] = len(set(filtered))

    return render_template("overview.html", stats=stats, user=user, building=building,
                           this_page=this_page, all_pages=all_pages, page=tlpage, rev_pending=[])
Esempio n. 6
0
def get_build_info(page=None, build_status=None, logged_in=False, search=None):
    """

    :param page:
    :param build_status:
    :param logged_in:
    :param search:
    :return:
    """
    if page is None or build_status is None:
        abort(500)

    if 'antergos' in build_status:
        build_status = 'completed'

    pkg_list = {}
    rev_pending = {}
    all_builds = None
    all_pages = 0

    try:
        all_builds = getattr(status, build_status)
    except Exception as err:
        logger.error('GET_BUILD_INFO - %s', err)
        abort(500)

    if all_builds:
        if search is not None:
            search_all_builds = [x for x in all_builds if x is not None and match_pkg_name_build_log(x, search)]
            logger.info('search_all_builds is %s', search_all_builds)
            all_builds = search_all_builds

        if all_builds:
            builds, all_pages = get_paginated(all_builds, 10, page, False)
            for bnum in builds:
                try:
                    bld_obj = build_obj.get_build_object(bnum=bnum)
                except Exception as err:
                    logger.error('Unable to ge build object - %s' % err)
                    continue

                all_info = dict(bnum=bld_obj.bnum, name=bld_obj.pkgname, version=bld_obj.version_str,
                                start=bld_obj.start_str, end=bld_obj.end_str, review_stat=bld_obj.review_status,
                                review_dev=bld_obj.review_dev, review_date=bld_obj.review_date)
                pkg_list[bnum] = all_info

                if logged_in and bld_obj.review_status == "pending":
                    rev_pending[bnum] = all_info

    return pkg_list, int(all_pages), rev_pending
Esempio n. 7
0
def build():
    now_building = status.now_building
    cont = status.container
    if cont:
        container = cont[:20]
    else:
        container = None
    bnum = status.building_num
    start = status.building_start
    try:
        bld_obj = build_obj.get_build_object(bnum=bnum)
        ver = bld_obj.version_str
    except Exception as err:
        logger.error(err)
        ver = ''

    return render_template("building.html", building=now_building, container=container, bnum=bnum,
                           start=start, ver=ver)
Esempio n. 8
0
def get_repo_info(repo=None, logged_in=False):
    """

    :param repo:
    :param logged_in:
    :return:
    """
    if repo is None:
        abort(500)
    pkg_list = {}
    p, a, rev_pending = get_build_info(1, repo, logged_in)

    logger.info('@@-antbs.py-@@ 295 | GET_REPO_INFO - CACHE CHECK FAILED. WE ARE NOT USING CACHED INFO')
    all_packages = glob.glob('/srv/antergos.info/repo/%s/x86_64/***.pkg.tar.xz' % repo)

    if all_packages is not None:
        for item in all_packages:
            # logger.info(item)
            item = item.split('/')[-1]
            item = re.search('^([a-z]|[0-9]|-|_)+(?=-\d|r|v)', item)

            item = item.group(0) or ''
            if not item or item == '':
                continue
            logger.info(item)
            pkg = package.get_pkg_object(item)
            builds = pkg.builds
            try:
                bnum = builds[0]
            except Exception:
                bnum = ''
            bld_obj = build_obj.get_build_object(bnum=bnum)
            all_info = dict(bnum=bnum, name=pkg.name, version=pkg.version_str, review_dev=bld_obj.review_dev,
                            review_stat=bld_obj.review_stat, review_date=bld_obj.review_date, pkgid=pkg.pkgid)
            pkg_list[pkg.pkgid] = all_info

    return pkg_list, rev_pending
Esempio n. 9
0
def process_and_save_build_metadata(pkg_obj=None):
    """

    :param pkg_obj:
    :return: :raise AttributeError:
    """
    if not pkg_obj:
        raise AttributeError

    status.current_status = 'Building %s' % pkg_obj.name
    status.now_building = pkg_obj.name
    logger.info('Building %s' % pkg_obj.name)
    bld_obj = build_obj.get_build_object(pkg_obj=pkg_obj)
    bld_obj.start_str = datetime.datetime.now().strftime("%m/%d/%Y %I:%M%p")
    status.building_num = bld_obj.bnum
    status.building_start = bld_obj.start_str
    build_id = bld_obj.bnum
    tlmsg = 'Build <a href="/build/%s">%s</a> for <strong>%s</strong> started.' % (build_id, build_id, pkg_obj.name)
    Timeline(msg=tlmsg, tl_type=3)
    pbuilds = pkg_obj.builds
    pbuilds.append(build_id)
    run_docker_clean(pkg_obj.name)

    return bld_obj
Esempio n. 10
0
def build():
    """


    :return:
    """
    now_building = status.now_building
    ver = ''
    bnum = ''
    start = ''
    cont = status.container
    if cont:
        container = cont[:20]
    else:
        container = None
    if not status.idle:
        bnum = status.building_num
        start = status.building_start
        if bnum and bnum != '':
            bld_obj = build_obj.get_build_object(bnum=bnum)
            ver = bld_obj.version_str

    return render_template("building.html", building=now_building, container=container, bnum=bnum,
                           start=start, ver=ver)
Esempio n. 11
0
def build_pkgs(last=False, pkg_info=None):
    if pkg_info is None:
        return False
    # Create our tmp directories
    result = os.path.join("/tmp", "result")
    cache = os.path.join("/var/tmp", "pkg_cache")
    for d in [result, cache]:
        if os.path.exists(d) and 'result' in d:
            shutil.rmtree(d)
            os.mkdir(d, 0o777)
        elif os.path.exists(d) and 'pkg_cache' in d:
            logger.info('@@-build_pkg.py-@@ 476 | Cleaning package cache....')
            db.set('building', 'Cleaning package cache.')
            for pcache in os.listdir(d):
                pcache = os.path.join(d, pcache)
                if not os.path.isdir(pcache):
                    logger.error('@@-build_pkg.py-@@ 479 | pcache is not a directory')
                    continue
                for pfile in os.listdir(pcache):
                    pname = re.search('^([a-z]|[0-9]|-|_)+(?=-\d|r|v)', pfile)
                    if not pname or pname == '':
                        continue
                    pname = pname.group(0)
                    pfile = os.path.join(pcache, pfile)
                    dtime = time.time()
                    if os.stat(pfile).st_mtime < (dtime - 7 * 86400) or status.all_packages().ismember(pname):
                        remove(pfile)
        else:
            os.mkdir(d, 0o777)
    dirs = ['/var/tmp/32build', '/var/tmp/32bit']
    for d in dirs:
        if os.path.exists(d):
            shutil.rmtree(d)
        os.mkdir(d, 0o777)
    # pkglist = db.lrange('queue', 0, -1)
    pkglist1 = ['1']
    in_dir_last = len([name for name in os.listdir(result)])
    db.set('pkg_count', in_dir_last)
    for i in range(len(pkglist1)):
        pkg = pkg_info.name
        if pkg and pkg is not None and pkg != '':
            pkgbuild_dir = pkg_info.build_path
            status.current_status = 'Building %s with makepkg' % pkg
            bld_obj = build_obj.get_build_object(pkg_obj=pkg_info)
            bld_obj.failed = False
            bld_obj.completed = False
            bld_obj.version_str = pkg_info.version_str
            bld_obj.start_str = datetime.datetime.now().strftime("%m/%d/%Y %I:%M%p")
            status.building_num = bld_obj.bnum
            status.building_start = bld_obj.start_str
            build_id = bld_obj.bnum
            tlmsg = 'Build <a href="/build/%s">%s</a> for <strong>%s</strong> started.' % (build_id, build_id, pkg)
            Timeline(msg=tlmsg, tl_type=3)
            pbuilds = pkg_info.builds()
            pbuilds.append(build_id)
            bld_obj.pkgname = pkg
            pkg_deps = pkg_info.depends() or []
            pkg_deps_str = ' '.join(pkg_deps)
            run_docker_clean(pkg)

            if pkg_info is not None and pkg_info.autosum == "True":
                build_env = ['_AUTOSUMS=True']
            else:
                build_env = ['_AUTOSUMS=False']
            if '/cinnamon/' in pkg_info.path:
                build_env.append('_ALEXPKG=True')
            else:
                build_env.append('_ALEXPKG=False')
            hconfig = docker_utils.create_pkgs_host_config(cache, pkgbuild_dir, result)
            try:
                container = doc.create_container("antergos/makepkg",
                                                 command="/makepkg/build.sh " + pkg_deps_str,
                                                 volumes=['/var/cache/pacman', '/makepkg', '/repo',
                                                          '/pkg', '/root/.gnupg', '/staging',
                                                          '/32bit', '/32build', '/result'],
                                                 environment=build_env, cpuset='0-3', name=pkg,
                                                 host_config=hconfig)
                if container.get('Warnings') and container.get('Warnings') != '':
                    logger.error(container.get('Warnings'))
            except Exception as err:
                logger.error('Create container failed. Error Msg: %s' % err)
                bld_obj.failed = True
                bld_obj.completed = False
                continue

            bld_obj.container = container.get('Id')

            try:
                doc.start(container.get('Id'))
                cont = bld_obj.container
                stream_process = Process(target=publish_build_ouput, args=(cont, bld_obj))
                stream_process.start()
                result = doc.wait(cont)
                if result is not 0:
                    bld_obj.failed = True
                    bld_obj.completed = False
                    logger.error('[CONTAINER EXIT CODE] Container %s exited. Return code was %s' % (pkg, result))
                else:
                    logger.info('[CONTAINER EXIT CODE] Container %s exited. Return code was %s' % (pkg, result))
                    bld_obj.failed = False
                    bld_obj.completed = True
            except Exception as err:
                logger.error('Start container failed. Error Msg: %s' % err)
                bld_obj.failed = True
                bld_obj.completed = False
                continue
            # db.publish('build-ouput', 'ENDOFLOG')
            # stream = doc.logs(container, stdout=True, stderr=True, timestamps=True)
            # log_stream = stream.split('\n')
            # db_filter_and_add(log_stream, this_log)

            # in_dir = len([name for name in os.listdir(result)])
            # last_count = int(db.get('pkg_count'))
            # logger.info('last count is %s %s' % (last_count, type(last_count)))
            # logger.info('in_dir is %s %s' % (in_dir, type(in_dir)))
            pkgs2sign = None
            if not bld_obj.failed:
                db.publish('build-output', 'Signing package..')
                pkgs2sign = glob.glob(
                    '/srv/antergos.info/repo/iso/testing/uefi/antergos-staging/x86_64/%s-***.xz' % pkg)
                pkgs2sign32 = glob.glob(
                    '/srv/antergos.info/repo/iso/testing/uefi/antergos-staging/i686/%s-***.xz' % pkg)
                pkgs2sign = pkgs2sign + pkgs2sign32
                logger.info('[PKGS TO SIGN] %s' % pkgs2sign)
                if pkgs2sign is not None and pkgs2sign != []:
                    try_sign = sign_pkgs.batch_sign(pkgs2sign)
                else:
                    try_sign = False
                if try_sign:
                    db.publish('build-output', 'Signature created successfully for %s' % pkg)
                    logger.info('[SIGN PKG] Signature created successfully for %s' % pkg)
                    db.publish('build-output', 'Updating staging repo database..')
                    update_main_repo(pkg, 'staging', bld_obj)
                else:
                    bld_obj.failed = True
                    bld_obj.completed = False

            if not bld_obj.failed:
                db.publish('build-output', 'Build completed successfully!')
                tlmsg = 'Build <a href="/build/%s">%s</a> for <strong>%s</strong> completed.' % (
                    build_id, build_id, pkg)
                Timeline(msg=tlmsg, tl_type=4)
                # db.incr('pkg_count', (in_dir - last_count))
                completed = status.completed()
                completed.rpush(build_id)
                bld_obj.review_stat = 'pending'
            else:
                tlmsg = 'Build <a href="/build/%s">%s</a> for <strong>%s</strong> failed.' % (build_id, build_id, pkg)
                Timeline(msg=tlmsg, tl_type=5)
                if pkgs2sign is not None:
                    for p in pkgs2sign:
                        remove(p)
                        remove(p + '.sig')
                failed = status.failed()
                failed.rpush(build_id)

            bld_obj.end_str = datetime.datetime.now().strftime("%m/%d/%Y %I:%M%p")

            status.container = ''
            status.building_num = ''
            status.building_start = ''

            if not bld_obj.failed:
                db.set('antbs:misc:cache_buster:flag', True)
                return True
            return False
Esempio n. 12
0
def set_pkg_review_result(bnum=None, dev=None, result=None):
    # TODO: This is garbage. Needs rewrite.
    """

    :param bnum:
    :param dev:
    :param result:
    :return:
    """
    if any(i is None for i in (bnum, dev, result)):
        abort(500)
    errmsg = dict(error=True, msg=None)
    dt = datetime.now().strftime("%m/%d/%Y %I:%M%p")
    if result in ['0', '1', '2', '3', '4']:
        msg = 'Please clear your browser cache, refresh the page, and try again.'
        errmsg.update(error=True, msg=msg)
        return errmsg
    try:
        bld_obj = build_obj.get_build_object(bnum=bnum)
        pkg_obj = package.Package(name=bld_obj.pkgname)
        if pkg_obj and build_obj:
            allowed = pkg_obj.allowed_in
            if 'main' not in allowed and result == 'passed':
                msg = '%s is not allowed in main repo.' % pkg_obj.pkgname
                errmsg.update(error=True, msg=msg)
                return errmsg
            else:
                bld_obj.review_dev = dev
                bld_obj.review_date = dt
                bld_obj.review_status = result

        if result == 'skip':
            errmsg = dict(error=False, msg=None)
            return errmsg

        pkg_files_64 = glob.glob('%s/%s-***' % (STAGING_64, pkg_obj.pkgname))
        pkg_files_32 = glob.glob('%s/%s-***' % (STAGING_32, pkg_obj.pkgname))
        pkg_files = pkg_files_64 + pkg_files_32

        if pkg_files or True:
            for f in pkg_files_64:
                logger.debug('f in pkg_files_64 fired!')
                if result == 'passed':
                    copy(f, MAIN_64)
                    copy(f, '/tmp')
                elif result == 'failed':
                    os.remove(f)
            for f in pkg_files_32:
                if result == 'passed':
                    copy(f, MAIN_32)
                    copy(f, '/tmp')
                elif result == 'failed':
                    os.remove(f)
            if result and result != 'skip':
                repo_queue.enqueue_call(builder.update_main_repo, (result, None, True, bld_obj.pkgname), timeout=9600)
                errmsg = dict(error=False, msg=None)

        else:
            logger.error('While moving to main, no packages were found to move.')
            err = 'While moving to main, no packages were found to move.'
            errmsg = dict(error=True, msg=err)

    except (OSError, Exception) as err:
        logger.error('@@-antbs.py-@@ | Error while moving to main: %s', err)
        err = str(err)
        errmsg = dict(error=True, msg=err)

    return errmsg