def check_deps(packages):
# TODO: Come up with more versitile solution. This will only help in the most basic situations.
pkgs = packages
queued = db.lrange('queue', 0, -1)
matches = []
for pkg in pkgs:
deps = db.lrange('pkg:%s:deps' % pkg, 0, -1)
if set(deps).intersection(set(queued)):
logger.info('CHECK DEPS: %s added to matches.' % pkg)
matches.append(pkg)
continue
return set(matches)
def check_deps(packages):
# TODO: Come up with more versitile solution. This will only help in the most basic situations.
pkgs = packages
queued = db.lrange('queue', 0, -1)
matches = []
for pkg in pkgs:
deps = db.lrange('pkg:%s:deps' % pkg, 0, -1)
if set(deps).intersection(set(queued)):
logger.info('CHECK DEPS: %s added to matches.' % pkg)
matches.append(pkg)
continue
return set(matches)
def get_log_stream(bnum=None):
#doc = docker.Client(base_url='unix://var/run/docker.sock', version='1.12', timeout=10)
is_idle = db.get('idle')
now_building = db.get('building')
container = db.get('container')
if bnum is not None:
log = db.lrange('build_log:%s:content' % bnum, 0, -1)
log_end = db.lrange('build_log:%s:content' % bnum, -1, -1)
nodata = ['data: Unable to retrieve build log.\n\n']
if not log:
for line in nodata:
yield line
yield 'data: ENDOFLOG\n\n'
else:
for line in log:
yield 'data: %s\n\n' % line
yield 'data: ENDOFLOG\n\n'
else:
nodata = ['data: There are no active builds.\n\n']
if is_idle == "True" or now_building == "Idle" or now_building == 'Initializing...':
app.logger.debug("No active container detected")
for line in nodata:
yield line
yield 'data: ENDOFLOG\n\n'
else:
#doclog = doc.logs(container, stdout=True, stderr=True, stream=True, timestamps=True)
proc = subprocess.Popen(['docker', 'logs', '--follow', '-t', container], stdout=subprocess.PIPE)
stream = iter(proc.stdout.readline, '')
nodup = set()
part2 = None
for line in stream:
if not line or line == '':
continue
line = line.rstrip()
end = line[20:]
if end not in nodup:
gevent.sleep(.05)
nodup.add(end)
line = line.replace("can't", "can not")
if len(line) > 210:
part1 = line[:210]
part2 = line[211:]
yield 'data: %s\n\n' % part1
continue
elif part2:
yield 'data: %s\n\n' % part2
part2 = None
continue
else:
yield 'data: %s\n\n' % line
yield 'data: ENDOFLOG\n\n'
def get_build_info(page=None, status=None):
if page is None or status is None:
abort(500)
try:
all_builds = db.lrange(status, 0, -1)
except Exception:
all_builds = None
pkg_list = []
all_pages = 1
if all_builds is not None:
#builds, all_pages = get_paginated(all_builds, 10, page)
for build in all_builds:
try:
pkg = db.get('build_log:%s:pkg' % build)
except Exception:
pass
name = db.get('pkg:%s:name' % pkg)
bnum = build
version = db.get('pkg:%s:version' % pkg)
start = db.get('build_log:%s:start' % bnum)
end = db.get('build_log:%s:end' % bnum)
all_info = dict(bnum=bnum, name=name, version=version, start=start, end=end)
pkg_info = {bnum: all_info}
pkg_list.append(pkg_info)
logger.info(pkg_list)
return pkg_list, all_pages
def handle_hook():
iso_flag = db.get('isoFlag')
if iso_flag == 'True':
archs = ['x86_64', 'i686']
for arch in archs:
db.rpush('queue', 'antergos-iso-%s' % arch)
version = datetime.datetime.now().strftime('%Y.%m.%d')
if not db.exists('pkg:antergos-iso-%s' % arch):
db.set('pkg:antergos-iso-%s' % arch, True)
db.set('pkg:antergos-iso-%s:name' % arch,
'antergos-iso-%s' % arch)
db.set('pkg:antergos-iso-%s:version' % arch, version)
build_iso()
else:
logger.info('Pulling changes from github.')
subprocess.call([
'git', 'clone', 'http://github.com/lots0logs/antergos-packages.git'
],
cwd='/opt')
subprocess.call(['chmod', '-R', '777', 'antergos-packages'],
cwd='/opt')
# Check database to see if packages exist and add them if necessary.
packages = db.lrange('queue', 0, -1)
logger.info('Checking database for packages.')
for package in packages:
version = get_pkgver(package)
depends = get_deps(package)
if not db.exists('pkg:%s' % package):
logger.info('%s not found in database, adding entry..' %
package)
db.set('pkg:%s' % package, True)
db.set('pkg:%s:name' % package, package)
for dep in depends:
db.rpush('pkg:%s:deps' % package, dep)
logger.info('Updating pkgver in databse for %s to %s' %
(package, version))
db.set('pkg:%s:version' % package, version)
logger.info(
'All queued packages are in the database, checking deps to determine build order.'
)
check = check_deps(packages)
if len(check) > 0:
for c in check:
logger.info(
'%s depends on a pkg in this build. Moving it to the end of the queue.'
% c)
db.lrem('queue', 0, c)
db.rpush('queue', c)
logger.info('Check deps complete. Starting build_pkgs')
build_pkgs()
def scheduled():
is_idle = db.get('idle')
try:
pkgs = db.lrange('queue', 0, -1)
except Exception:
pkgs = None
building = db.get('building')
the_queue = {}
if pkgs is not None:
for pak in pkgs:
name = db.get('pkg:%s:name' % pak)
version = db.get('pkg:%s:version' % pak)
all_info = dict(name=name, version=version)
the_queue[pak] = all_info
return render_template("scheduled.html", idle=is_idle, building=building, queue=the_queue)
def homepage():
is_idle = db.get('idle')
check_stats = ['queue', 'completed', 'failed']
stats = {}
for stat in check_stats:
res = db.llen(stat)
stats[stat] = res
if stat is not "queue":
builds = db.lrange(stat, 0, -1)
within = []
for build in builds:
end = db.get('build_log:%s:start' % build)
end_fmt = datetime.strptime(end, '%m/%d/%Y %I:%M%p')
if (datetime.now() - end_fmt) < timedelta(hours=72):
within.append(build)
stats[stat] = len(within)
x86_64 = glob.glob('/srv/antergos.info/repo/iso/testing/uefi/antergos/x86_64/*.pkg.tar.xz')
i686 = glob.glob('/srv/antergos.info/repo/iso/testing/uefi/antergos/i686/*.pkg.tar.xz')
stats['repo'] = len(set(x86_64 + i686))
return render_template("overview.html", idle=is_idle, stats=stats)
def handle_hook():
iso_flag = db.get('isoFlag')
if iso_flag == 'True':
archs = ['x86_64', 'i686']
for arch in archs:
db.rpush('queue', 'antergos-iso-%s' % arch)
version = datetime.datetime.now().strftime('%Y.%m.%d')
if not db.exists('pkg:antergos-iso-%s' % arch):
db.set('pkg:antergos-iso-%s' % arch, True)
db.set('pkg:antergos-iso-%s:name' % arch, 'antergos-iso-%s' % arch)
db.set('pkg:antergos-iso-%s:version' % arch, version)
build_iso()
else:
logger.info('Pulling changes from github.')
subprocess.call(['git', 'clone', 'http://github.com/lots0logs/antergos-packages.git'], cwd='/opt')
subprocess.call(['chmod', '-R', '777', 'antergos-packages'], cwd='/opt')
# Check database to see if packages exist and add them if necessary.
packages = db.lrange('queue', 0, -1)
logger.info('Checking database for packages.')
for package in packages:
version = get_pkgver(package)
depends = get_deps(package)
if not db.exists('pkg:%s' % package):
logger.info('%s not found in database, adding entry..' % package)
db.set('pkg:%s' % package, True)
db.set('pkg:%s:name' % package, package)
for dep in depends:
db.rpush('pkg:%s:deps' % package, dep)
logger.info('Updating pkgver in databse for %s to %s' % (package, version))
db.set('pkg:%s:version' % package, version)
logger.info('All queued packages are in the database, checking deps to determine build order.')
check = check_deps(packages)
if len(check) > 0:
for c in check:
logger.info('%s depends on a pkg in this build. Moving it to the end of the queue.' % c)
db.lrem('queue', 0, c)
db.rpush('queue', c)
logger.info('Check deps complete. Starting build_pkgs')
build_pkgs()
def build_pkgs():
# Initiate communication with docker daemon
try:
doc = docker.Client(base_url='unix://var/run/docker.sock', version='1.12', timeout=10)
# doc.build(path=DOC_DIR, tag="arch-devel", quiet=False, timeout=None)
except Exception as err:
logger.error("Cant connect to Docker daemon. Error msg: %s", err)
# Create our tmp directories
repo = os.path.join("/tmp", "staging")
cache = os.path.join("/tmp", "pkg_cache")
for d in [repo, cache]:
if not os.path.exists(d):
os.mkdir(d, 0o777)
db.set('pkg_count', '0')
pkglist = db.lrange('queue', 0, -1)
for i in range(len(pkglist)):
failed = False
pkg = db.lpop('queue')
if pkg is None or pkg == '':
continue
logger.info('Building %s' % pkg)
db.incr('build_number')
dt = datetime.datetime.now().strftime("%m/%d/%Y %I:%M%p")
build_id = db.get('build_number')
db.set('building_num', build_id)
this_log = 'build_log:%s' % build_id
db.set(this_log, True)
db.rpush('pkg:%s:build_logs' % pkg, build_id)
db.set('%s:start' % this_log, dt)
db.set('building_start', dt)
db.set('%s:pkg' % this_log, pkg)
db.set('building', pkg)
pkgdir = os.path.join(REPO_DIR, pkg)
pkg_deps = db.lrange('pkg:%s:deps' % pkg, 0, -1)
pkg_deps_str = ' '.join(pkg_deps)
logger.info('pkg_deps_str is %s' % pkg_deps_str)
try:
container = doc.create_container("antergos/makepkg", command=["/makepkg/build.sh", pkg_deps_str], name=pkg,
volumes=['/var/cache/pacman', '/makepkg', '/repo', '/pkg', '/root/.gnupg',
'/staging'])
except Exception as err:
logger.error('Create container failed. Error Msg: %s' % err)
failed = True
continue
db.set('container', container.get('Id'))
try:
doc.start(container, binds={
cache:
{
'bind': '/var/cache/pacman',
'ro': False
},
DOC_DIR:
{
'bind': '/makepkg',
'ro': True
},
repo:
{
'bind': '/staging',
'ro': False
},
pkgdir:
{
'bind': '/pkg',
'ro': False
},
'/root/.gnupg':
{
'bind': '/root/.gnupg',
'ro': False
},
'/srv/antergos.info/repo/iso/testing/uefi/antergos/':
{
'bind': '/repo',
'ro': False
}
})
doc.wait(container)
except Exception as err:
logger.error('Start container failed. Error Msg: %s' % err)
failed = True
continue
stream = doc.logs(container, stdout=True, stderr=True, timestamps=True)
log_stream = stream.split('\n')
db_filter_and_add(log_stream, this_log)
in_dir = len([name for name in os.listdir(repo)])
last_count = int(db.get('pkg_count'))
logger.info('last count is %s %s' % (last_count, type(last_count)))
logger.info('in_dir is %s %s' % (in_dir, type(in_dir)))
if in_dir > last_count:
db.incr('pkg_count', (in_dir - last_count))
db.rpush('completed', build_id)
db.set('%s:result' % this_log, 'completed')
else:
logger.error('No package found after container exit.')
failed = True
db.set('%s:result' % this_log, 'failed')
db.rpush('failed', build_id)
doc.remove_container(container)
end = datetime.datetime.now().strftime("%m/%d/%Y %I:%M%p")
db.set('%s:end' % this_log, end)
logger.info('Moving pkgs into repo and updating repo database')
try:
repo_container = doc.create_container("lots0logs/makepkg", command="/makepkg/repo_expect.sh",
volumes=['/var/cache/pacman', '/makepkg', '/repo', '/root/.gnupg',
'/staging'])
except Exception as err:
logger.error('Create container failed. Error Msg: %s' % err)
try:
doc.start(repo_container, binds={
cache:
{
'bind': '/var/cache/pacman',
'ro': False
},
DOC_DIR:
{
'bind': '/makepkg',
'ro': True
},
repo:
{
'bind': '/staging',
'ro': False
},
'/root/.gnupg':
{
'bind': '/root/.gnupg',
'ro': False
},
'/srv/antergos.info/repo/iso/testing/uefi/antergos/':
{
'bind': '/repo',
'ro': False
}
})
doc.wait(repo_container)
except Exception as err:
logger.error('Start container failed. Error Msg: %s' % err)
doc.remove_container(repo_container)
try:
shutil.rmtree(repo)
shutil.rmtree(cache)
shutil.rmtree('/opt/antergos-packages')
except Exception:
pass
db.set('idle', "True")
db.set('building', 'Idle')
db.set('container', '')
db.set('building_num', '')
db.set('building_start', '')
logger.info('All builds completed. Repo has been updated.')
def build_pkgs():
# Initiate communication with docker daemon
try:
doc = docker.Client(base_url='unix://var/run/docker.sock',
version='1.12',
timeout=10)
# doc.build(path=DOC_DIR, tag="arch-devel", quiet=False, timeout=None)
except Exception as err:
logger.error("Cant connect to Docker daemon. Error msg: %s", err)
# Create our tmp directories
repo = os.path.join("/tmp", "staging")
cache = os.path.join("/tmp", "pkg_cache")
for d in [repo, cache]:
if not os.path.exists(d):
os.mkdir(d, 0o777)
db.set('pkg_count', '0')
pkglist = db.lrange('queue', 0, -1)
for i in range(len(pkglist)):
failed = False
pkg = db.lpop('queue')
if pkg is None or pkg == '':
continue
logger.info('Building %s' % pkg)
db.incr('build_number')
dt = datetime.datetime.now().strftime("%m/%d/%Y %I:%M%p")
build_id = db.get('build_number')
db.set('building_num', build_id)
this_log = 'build_log:%s' % build_id
db.set(this_log, True)
db.rpush('pkg:%s:build_logs' % pkg, build_id)
db.set('%s:start' % this_log, dt)
db.set('building_start', dt)
db.set('%s:pkg' % this_log, pkg)
db.set('building', pkg)
pkgdir = os.path.join(REPO_DIR, pkg)
pkg_deps = db.lrange('pkg:%s:deps' % pkg, 0, -1)
pkg_deps_str = ' '.join(pkg_deps)
logger.info('pkg_deps_str is %s' % pkg_deps_str)
try:
container = doc.create_container(
"antergos/makepkg",
command=["/makepkg/build.sh", pkg_deps_str],
name=pkg,
volumes=[
'/var/cache/pacman', '/makepkg', '/repo', '/pkg',
'/root/.gnupg', '/staging'
])
except Exception as err:
logger.error('Create container failed. Error Msg: %s' % err)
failed = True
continue
db.set('container', container.get('Id'))
try:
doc.start(container,
binds={
cache: {
'bind': '/var/cache/pacman',
'ro': False
},
DOC_DIR: {
'bind': '/makepkg',
'ro': True
},
repo: {
'bind': '/staging',
'ro': False
},
pkgdir: {
'bind': '/pkg',
'ro': False
},
'/root/.gnupg': {
'bind': '/root/.gnupg',
'ro': False
},
'/srv/antergos.info/repo/iso/testing/uefi/antergos/':
{
'bind': '/repo',
'ro': False
}
})
doc.wait(container)
except Exception as err:
logger.error('Start container failed. Error Msg: %s' % err)
failed = True
continue
stream = doc.logs(container, stdout=True, stderr=True, timestamps=True)
log_stream = stream.split('\n')
db_filter_and_add(log_stream, this_log)
in_dir = len([name for name in os.listdir(repo)])
last_count = int(db.get('pkg_count'))
logger.info('last count is %s %s' % (last_count, type(last_count)))
logger.info('in_dir is %s %s' % (in_dir, type(in_dir)))
if in_dir > last_count:
db.incr('pkg_count', (in_dir - last_count))
db.rpush('completed', build_id)
db.set('%s:result' % this_log, 'completed')
else:
logger.error('No package found after container exit.')
failed = True
db.set('%s:result' % this_log, 'failed')
db.rpush('failed', build_id)
doc.remove_container(container)
end = datetime.datetime.now().strftime("%m/%d/%Y %I:%M%p")
db.set('%s:end' % this_log, end)
logger.info('Moving pkgs into repo and updating repo database')
try:
repo_container = doc.create_container(
"lots0logs/makepkg",
command="/makepkg/repo_expect.sh",
volumes=[
'/var/cache/pacman', '/makepkg', '/repo', '/root/.gnupg',
'/staging'
])
except Exception as err:
logger.error('Create container failed. Error Msg: %s' % err)
try:
doc.start(repo_container,
binds={
cache: {
'bind': '/var/cache/pacman',
'ro': False
},
DOC_DIR: {
'bind': '/makepkg',
'ro': True
},
repo: {
'bind': '/staging',
'ro': False
},
'/root/.gnupg': {
'bind': '/root/.gnupg',
'ro': False
},
'/srv/antergos.info/repo/iso/testing/uefi/antergos/': {
'bind': '/repo',
'ro': False
}
})
doc.wait(repo_container)
except Exception as err:
logger.error('Start container failed. Error Msg: %s' % err)
doc.remove_container(repo_container)
try:
shutil.rmtree(repo)
shutil.rmtree(cache)
shutil.rmtree('/opt/antergos-packages')
except Exception:
pass
db.set('idle', "True")
db.set('building', 'Idle')
db.set('container', '')
db.set('building_num', '')
db.set('building_start', '')
logger.info('All builds completed. Repo has been updated.')