def get_jobs( self ): ''' returns a list of new jobdescriptions ''' ret = [] # make sure we can exit safely if the web ui is down try: json_data = get_url( 'http://localhost:8080/stat' ) except: return ret temp_json = bytes.decode( json_data ) print( 'temp_json is:', temp_json ) print( 'json_data is:', json_data ) remote_pkgs = json.loads( temp_json ) makepkg_cmd = [ 'makepkg', '--source', '--skipinteg' ] for rpkg in remote_pkgs: # copy over the sources to a temp directory #orig_dir = os.path.join( '/var/abs/core', rpkg[ 'pkgname' ] ) dest_dir = os.path.join( '/tmp', rpkg[ 'uuid' ] ) # check if we already did this if os.path.exists( dest_dir ): continue svnco_cmd = [ 'svn', 'checkout', 'svn://svn.archlinux.org/packages/' + rpkg[ 'pkgname' ] + '/trunk', rpkg[ 'uuid' ] ] proc = subprocess.Popen( svnco_cmd, cwd='/tmp' ) proc.wait() #shutil.copytree( orig_dir, dest_dir ) pkgbuild_path = os.path.join( '/tmp', rpkg[ 'uuid' ], 'PKGBUILD' ) pkgbuild_data = pacman.load( pkgbuild_path ) # build the .src.tar.gz file proc = subprocess.Popen( makepkg_cmd, cwd=dest_dir ) proc.wait() # find the resulting .src.tar.gz file getsrc = glob.glob( os.path.join( dest_dir, '*.src.tar.gz' ) ) print( 'glob returned' + str( getsrc ) ) if len( getsrc ) != 1: print( 'error, not enough, or too many srcpkgs detected in Web' ) # TODO need to create a build log with a message from the scm telling what happened self.local_state.create_empty_job( rpkg[ 'uuid' ], 'failed' ) continue # get the sha256sum of the file sha256sum = sha256sum_file( getsrc[0] ) # move the srcpkg to the final resting place srcpkg_path = os.path.join( self.master_root, rpkg[ 'uuid' ] ) os.makedirs( srcpkg_path, exist_ok=True ) srcpkg_path = os.path.join( srcpkg_path, rpkg[ 'uuid' ] + '.src.tar.gz' ) shutil.move( getsrc[0], srcpkg_path ) # add the final jobdescription to the list jd = JobDescription( rpkg[ 'uuid' ], rpkg[ 'pkgname' ], sha256sum, pkgbuild_data[ 'arch' ] ) ret.append( jd ) return ret
# Go through each package, get the info, and apply the rules for package in packages: extracted = 0 if not os.access(package, os.R_OK): print "Error: Problem reading " + package usage() if package[-7:] == '.tar.gz': pkgtar = verify_package(package) if not pkgtar: print "Error: " + package + " is not a package" if len(packages) > 1: continue pkginfo = pacman.load(package) # No rules selected? Then select them all! if active_modules == []: active_modules = modules # Loop through each one, load them apply if possible for i in active_modules: cur_class = __import__('Namcap.' + i, globals(), locals(), [Namcap]) pkg = cur_class.package() ret = [[],[],[]] if pkg.type() == "tarball": if pkg.prereq() == "extract": # If it's not extracted, then extract it and then analyze the package if not extracted: os.mkdir(sandbox_directory)
def load(name, path=None): if not pkgcache.has_key(name): pkgcache[name] = pacman.load(name) return pkgcache[name]
def analyze(self, pkginfo, data): liblist = [{}, {}] dependlist = {} smartdepend = {} smartprovides = {} covereddepend = {} pkgcovered = {} ret = [[], [], []] filllibcache() os.environ['LC_ALL'] = 'C' os.path.walk(data, scanlibs, liblist) # Ldd all the files and find all the link and script dependencies dependlist, tmpret = finddepends(liblist[0]) # Handle "no package associated" errors for i in tmpret: ret[1].append(i) # Do the script handling stuff for i, v in liblist[1].iteritems(): if not dependlist.has_key(i): dependlist[i] = {} for j in v.keys(): dependlist[i][j] = 1 files = [x[len(data) + 1:] for x in v.keys()] ret[2].append('Script link detected (' + i + ') in file ' + str(files)) # Remove the package name from that list, we can't depend on ourselves. if dependlist.has_key(pkginfo.name): del dependlist[pkginfo.name] # Do the info stuff for i, v in dependlist.iteritems(): if type(v) == dict: files = [x[len(data) + 1:] for x in v.keys()] ret[2].append('File ' + str(files) + ' link-level dependence on ' + i) # Check for packages in testing if os.path.isdir('/var/lib/pacman/sync/testing'): for i in dependlist.keys(): p = pacman.load(i, '/var/lib/pacman/sync/testing/') q = load(i) if p != None and q != None and p.version == q.version: ret[1].append('Dependency ' + i + ' on your system is a testing release') # Find all the covered dependencies from the PKGBUILD pkgdepend = {} if hasattr(pkginfo, 'depends'): for i in pkginfo.depends: pkgdepend[i] = 1 # Include the optdepends from the PKGBUILD if hasattr(pkginfo, 'optdepends'): for i in pkginfo.optdepends: pkgdepend[i] = 1 getcovered(None, pkgdepend, pkgcovered) # Do tree walking to find all the non-leaves (branches?) getcovered(None, dependlist, covereddepend) for i in covereddepend.keys(): ret[2].append( 'Dependency covered by dependences from link dependence (' + i + ')') # Set difference them to find the leaves for i in dependlist.keys(): if not i in covereddepend.keys(): smartdepend[i] = 1 # Get the provides so we can reference them later getprovides(dependlist, smartprovides) # Do the actual message outputting stuff for i in smartdepend.keys(): # If (i is not in the PKGBUILD's dependencies # and i isn't the package name # and ((there are provides for i # and those provides aren't included in the package's dependencies) # or there are no provides for i)) all_dependencies = getattr(pkginfo, 'depends', []) + getattr( pkginfo, 'optdepends', []) if (i not in all_dependencies and i != pkginfo.name and ((smartprovides.has_key(i) and len([ c for c in smartprovides[i] if c in pkgcovered.keys() ]) == 0) or not smartprovides.has_key(i))): if type(dependlist[i]) == dict: ret[0].append( 'Dependency detected and not included (' + i + ') from files ' + str([x[len(data) + 1:] for x in dependlist[i].keys()])) else: ret[0].append('Dependency detected and not included (' + i + ')') if hasattr(pkginfo, 'depends'): for i in pkginfo.depends: if covereddepend.has_key(i) and dependlist.has_key(i): ret[1].append( 'Dependency included but already satisfied (' + i + ')') # if i is not in the depends as we see them and it's not in any of the provides from said depends elif not smartdepend.has_key(i) and i not in [ y for x in smartprovides.values() for y in x ]: ret[1].append('Dependency included and not needed (' + i + ')') ret[2].append('Depends as namcap sees them: depends=(' + ' '.join(smartdepend.keys()) + ')') return ret
def analyze(self, pkginfo, data): liblist = [{},{}] dependlist = {} smartdepend = {} smartprovides = {} covereddepend = {} pkgcovered = {} ret = [[],[],[]] filllibcache() os.environ['LC_ALL'] = 'C' os.path.walk(data, scanlibs, liblist) # Ldd all the files and find all the link and script dependencies dependlist, tmpret = finddepends(liblist[0]) # Handle "no package associated" errors for i in tmpret: ret[1].append(i) # Do the script handling stuff for i, v in liblist[1].iteritems(): if not dependlist.has_key(i): dependlist[i] = {} for j in v.keys(): dependlist[i][j] = 1 files = [x[len(data)+1:] for x in v.keys()] ret[2].append(("script-link-detected %s in %s", (i, str(files)))) # Remove the package name from that list, we can't depend on ourselves. if dependlist.has_key(pkginfo.name): del dependlist[pkginfo.name] # Do the info stuff for i, v in dependlist.iteritems(): if type(v) == dict: files = [x[len(data)+1:] for x in v.keys()] ret[2].append(("link-level-dependence %s on %s", (str(files), i))) # Check for packages in testing if os.path.isdir('/var/lib/pacman/sync/testing'): for i in dependlist.keys(): p = pacman.load(i, '/var/lib/pacman/sync/testing/') q = load(i) if p != None and q != None and p.version == q.version: ret[1].append(("dependency-is-testing-release %s", i)) # Find all the covered dependencies from the PKGBUILD pkgdepend = {} if hasattr(pkginfo, 'depends'): for i in pkginfo.depends: pkgdepend[i] = 1 # Include the optdepends from the PKGBUILD if hasattr(pkginfo, 'optdepends'): for i in pkginfo.optdepends: pkgdepend[i] = 1 getcovered(None, pkgdepend, pkgcovered) # Do tree walking to find all the non-leaves (branches?) getcovered(None, dependlist, covereddepend) for i in covereddepend.keys(): ret[2].append(("dependency-covered-by-link-dependence %s", i)) # Set difference them to find the leaves for i in dependlist.keys(): if not i in covereddepend.keys(): smartdepend[i] = 1 # Get the provides so we can reference them later getprovides(dependlist, smartprovides) # Do the actual message outputting stuff for i in smartdepend.keys(): # If (i is not in the PKGBUILD's dependencies # and i isn't the package name # and ((there are provides for i # and those provides aren't included in the package's dependencies) # or there are no provides for i)) all_dependencies = getattr(pkginfo, 'depends', []) + getattr(pkginfo, 'optdepends', []) if (i not in all_dependencies and i != pkginfo.name and ((smartprovides.has_key(i) and len([c for c in smartprovides[i] if c in pkgcovered.keys()]) == 0) or not smartprovides.has_key(i))): if type(dependlist[i]) == dict: ret[0].append(("dependency-detected-not-included %s from files %s", (i, str([x[len(data)+1:] for x in dependlist[i].keys()])) )) else: ret[0].append(("dependency-detected-not-included %s", i)) if hasattr(pkginfo, 'depends'): for i in pkginfo.depends: if covereddepend.has_key(i) and dependlist.has_key(i): ret[1].append(("dependency-already-satisfied %s", i)) # if i is not in the depends as we see them and it's not in any of the provides from said depends elif not smartdepend.has_key(i) and i not in [y for x in smartprovides.values() for y in x]: ret[1].append(("dependency-not-needed %s", i)) ret[2].append(("depends-by-namcap-sight depends=(%s)", ' '.join(smartdepend.keys()) )) return ret
# Go through each package, get the info, and apply the rules for package in packages: extracted = 0 if not os.access(package, os.R_OK): print "Error: Problem reading " + package usage() if package[-7:] == '.tar.gz': pkgtar = verify_package(package) if not pkgtar: print "Error: " + package + " is not a package" if len(packages) > 1: continue pkginfo = pacman.load(package) # No rules selected? Then select them all! if active_modules == []: active_modules = modules # Loop through each one, load them apply if possible for i in active_modules: cur_class = __import__('Namcapp.' + i, globals(), locals(), [Namcap]) pkg = cur_class.package() ret = [[],[],[]] if pkg.type() == "tarball": tar_tvf = [] get_output = subprocess.Popen("tar -tvf" + package, shell=True, stdout = subprocess.PIPE, stderr = subprocess.subprocess.PIPE).communicate() split_output = get_output[0].split('\n')
def analyze(self, pkginfo, data, tar_tvf): liblist = [{},{}] dependlist = {} smartdepend = {} smartprovides = {} covereddepend = {} pkgcovered = {} ret = [[],[],[]] extract_required(tar_tvf) filllibcache() os.environ['LC_ALL'] = 'C' os.path.walk(data, scanlibs, liblist) # Ldd all the files and find all the link and script dependencies dependlist, tmpret = finddepends(liblist[0]) # it is liblist[0] because [0], the first dict, contains shared libs, and the second contains script dependencies # Handle "no package associated" errors for i in tmpret: ret[1].append(i) # Do the script handling stuff for i, v in liblist[1].iteritems(): if not dependlist.has_key(i): dependlist[i] = {} for j in v.keys(): dependlist[i][j] = 1 files = [x[len(data)+1:] for x in v.keys()] ret[2].append('Script link detected (' + i + ') in file ' + str(files)) # Remove the package name from that list, we can't depend on ourselves. if dependlist.has_key(pkginfo.name): del dependlist[pkginfo.name] # Do the info stuff # THIS IS NEXT for i, v in dependlist.iteritems(): if type(v) == dict: files = [x[len(data)+1:] for x in v.keys()] ret[2].append('File '+ str(files) +' link-level dependence on ' + i) # Check for packages in testing if os.path.isdir('/var/lib/pacman/sync/testing'): for i in dependlist.keys(): p = pacman.load(i, '/var/lib/pacman/sync/testing/') q = load(i) if p != None and q != None and p.version == q.version: ret[1].append('Dependency ' + i + ' on your system is a testing release') # Find all the covered dependencies from the PKGBUILD pkgdepend = {} if hasattr(pkginfo, 'depends'): for i in pkginfo.depends: pkgdepend[i] = 1 # Include the optdepends from the PKGBUILD if hasattr(pkginfo, 'optdepends'): for i in pkginfo.optdepends: pkgdepend[i] = 1 getcovered(None, pkgdepend, pkgcovered) # Do tree walking to find all the non-leaves (branches?) #find all dependencies of dependencies of the shared libraries found from the actual files getcovered(None, dependlist, covereddepend) for i in covereddepend.keys(): ret[2].append('Dependency covered by dependences from link dependence (' + i + ')') # Set difference them to find the leaves for i in dependlist.keys(): if not i in covereddepend.keys(): smartdepend[i] = 1 # Get the provides so we can reference them later getprovides(dependlist, smartprovides) # Do the actual message outputting stuff for i in smartdepend.keys(): # If (i is not in the PKGBUILD's dependencies # and i isn't the package name # and ((there are provides for i # and those provides aren't included in the package's dependencies) # or there are no provides for i)) all_dependencies = getattr(pkginfo, 'depends', []) + getattr(pkginfo, 'optdepends', []) if (i not in all_dependencies and i != pkginfo.name and ((smartprovides.has_key(i) and len([c for c in smartprovides[i] if c in pkgcovered.keys()]) == 0) or not smartprovides.has_key(i))): if type(dependlist[i]) == dict: ret[0].append('Dependency detected and not included ('+i+') from files '+str([x[len(data)+1:] for x in dependlist[i].keys()])) else: ret[0].append('Dependency detected and not included ('+i+')') if hasattr(pkginfo, 'depends'): for i in pkginfo.depends: if covereddepend.has_key(i) and dependlist.has_key(i): ret[1].append('Dependency included but already satisfied ('+i+')') # if i is not in the depends as we see them and it's not in any of the provides from said depends elif not smartdepend.has_key(i) and i not in [y for x in smartprovides.values() for y in x]: ret[1].append('Dependency included and not needed ('+i+')') ret[2].append('Depends as namcap sees them: depends=('+ ' '.join(smartdepend.keys())+')') return ret