def get_jobs( self ):
''' returns a list of new jobdescriptions '''
ret = []
# make sure we can exit safely if the web ui is down
try:
json_data = get_url( 'http://localhost:8080/stat' )
except:
return ret
temp_json = bytes.decode( json_data )
print( 'temp_json is:', temp_json )
print( 'json_data is:', json_data )
remote_pkgs = json.loads( temp_json )
makepkg_cmd = [ 'makepkg', '--source', '--skipinteg' ]
for rpkg in remote_pkgs:
# copy over the sources to a temp directory
#orig_dir = os.path.join( '/var/abs/core', rpkg[ 'pkgname' ] )
dest_dir = os.path.join( '/tmp', rpkg[ 'uuid' ] )
# check if we already did this
if os.path.exists( dest_dir ):
continue
svnco_cmd = [ 'svn', 'checkout', 'svn://svn.archlinux.org/packages/' + rpkg[ 'pkgname' ] + '/trunk', rpkg[ 'uuid' ] ]
proc = subprocess.Popen( svnco_cmd, cwd='/tmp' )
proc.wait()
#shutil.copytree( orig_dir, dest_dir )
pkgbuild_path = os.path.join( '/tmp', rpkg[ 'uuid' ], 'PKGBUILD' )
pkgbuild_data = pacman.load( pkgbuild_path )
# build the .src.tar.gz file
proc = subprocess.Popen( makepkg_cmd, cwd=dest_dir )
proc.wait()
# find the resulting .src.tar.gz file
getsrc = glob.glob( os.path.join( dest_dir, '*.src.tar.gz' ) )
print( 'glob returned' + str( getsrc ) )
if len( getsrc ) != 1:
print( 'error, not enough, or too many srcpkgs detected in Web' )
# TODO need to create a build log with a message from the scm telling what happened
self.local_state.create_empty_job( rpkg[ 'uuid' ], 'failed' )
continue
# get the sha256sum of the file
sha256sum = sha256sum_file( getsrc[0] )
# move the srcpkg to the final resting place
srcpkg_path = os.path.join( self.master_root, rpkg[ 'uuid' ] )
os.makedirs( srcpkg_path, exist_ok=True )
srcpkg_path = os.path.join( srcpkg_path, rpkg[ 'uuid' ] + '.src.tar.gz' )
shutil.move( getsrc[0], srcpkg_path )
# add the final jobdescription to the list
jd = JobDescription( rpkg[ 'uuid' ], rpkg[ 'pkgname' ], sha256sum, pkgbuild_data[ 'arch' ] )
ret.append( jd )
return ret
def get_jobs( self ):
''' returns a list of new jobdescriptions '''
ret = []
svnup_cmd = [ '/usr/bin/svn', 'up' ]
pkgs = set()
with subprocess.Popen( svnup_cmd, cwd=self.svn_root, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) as proc:
lines = proc.communicate()[0].splitlines()
# find unique pkgnames
for line in lines[0:-1]:
pkgname = bytes.decode(line).split('/')[0].split()[1]
pkgs.add( pkgname )
makepkg_cmd = [ 'makepkg', '--source', '--skipinteg' ]
for pkg in pkgs:
new_ujid = str( uuid.uuid4() )
# create a srcpkg
print( 'getting package from svn' )
print( pkg )
pkg_path = os.path.join( self.svn_root, pkg + '/trunk' )
proc = subprocess.Popen( makepkg_cmd, cwd=pkg_path ) #, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
proc.wait()
print( 'process returned ' + str( proc.returncode ) )
srcpkg_path = os.path.join( self.master_root, new_ujid )
os.makedirs( srcpkg_path, exist_ok=True )
srcpkg_path = os.path.join( srcpkg_path, new_ujid + '.src.tar.gz' )
getsrc = glob.glob( os.path.join( pkg_path, pkg + '*.src.tar.gz' ) )
print( 'glob returned' + str( getsrc ) )
if len( getsrc ) != 1:
print( 'error moving srcpkgs, not enough, or too many detected' )
continue
shutil.move( getsrc[0], srcpkg_path )
sha256sum = sha256sum_file( srcpkg_path )
# job description: ujid, cur_pkgname, pkgsrc, sha256sum of srcpkg, architecture to build (x86_64,i686,any)
jd = JobDescription( new_ujid, pkgname, sha256sum, 'x86_64' )
ret.append( jd )
return ret
def worker( worker_id, local_state, config ):
''' worker where the grunt work takes place '''
builder_root = config['builder_root']
worker_chroot = config[ 'chroot_root' ] + str( worker_id )
while 1:
current_job = local_state.get_pending_job()
# update job state here (running)
local_state.set_status( current_job.ujid, 'inprogress' )
job_path = os.path.join( builder_root, str(current_job.ujid) )
pkgsrc_path = os.path.join( job_path, current_job.package_name + '.tar.gz' )
pkg_path = os.path.join( job_path, current_job.package_name )
os.makedirs( job_path, exist_ok=True )
# TODO: implement when we start using https
# need to make sure that urlretrieve overwrites if existing file with same name is found
# "If the URL points to a local file, or a valid cached copy of the object exists, the object is not copied."
# http://docs.python.org/py3k/library/urllib.request.html#urllib.request.urlretrieve
srcpkg_url = 'http://' + config[ 'master' ] + ':' + str( config[ 'master_port' ] ) + '/' + current_job.ujid + '/' + current_job.ujid + '.src.tar.gz'
# TODO: wrap this in a protocol.py function
urllib.request.urlretrieve( srcpkg_url, pkgsrc_path )
# extract
temp_tar = tarfile.open( pkgsrc_path )
temp_tar.extractall( job_path )
# chroot
errors = False
for arch in current_job.architecture:
if arch == 'i686':
chroot_cmd = [ 'sudo', 'extra-i686-build', '-r', worker_chroot ]
else:
# handle x86_64 & any
chroot_cmd = [ 'sudo', 'extra-x86_64-build', '-r', worker_chroot ]
with subprocess.Popen( chroot_cmd, cwd=pkg_path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) as proc:
log_path = os.path.join( job_path, 'build_log' )
with open( log_path, 'ab' ) as f:
f.write( proc.communicate()[0] )
return_code = proc.returncode
# if failed, just ignore the rest of the code
if return_code != 0:
local_state.set_status( current_job.ujid, 'failed' )
errors = True
break
if errors:
continue
# move to final destination
getsrc = glob.glob( os.path.join( pkg_path, '*.pkg.tar.[gx]z' ) )
for pkg in getsrc:
shutil.move( pkg, job_path )
getsrc = glob.glob( os.path.join( job_path, '*.pkg.tar.[gx]z' ) )
# update list of packages
results = list( map( (lambda x: x.split('/')[-1]), getsrc ) )
temp = [{ 'package' : i, 'sha256sum' : sha256sum_file( job_path + '/' + i ) } for i in results]
local_state.set_packages( current_job.ujid, temp )
local_state.set_status( current_job.ujid, 'done' )
