class RemoteContainer: def __init__(self, containername, configBuildMachine, logger, packageSrcPath, containertype): self.hostname = containername self.staticMachine = (True if ('static' in configBuildMachine and configBuildMachine['static'] == "t") else False) self.port="22" if configBuildMachine['port'] is not None: self.port=str(configBuildMachine['port']) self.cid=10 if configBuildMachine['cid'] is not None: self.cid=configBuildMachine['cid'] self.containername = str(self.cid).zfill(3) + "-" + containername self.containerIP=socket.gethostbyname(self.hostname) self.containerPort=str(2000+int(self.cid)) if configBuildMachine['local'] is not None and configBuildMachine['local'] == "t": # the host server for the build container is actually hosting the LBS application as well # or the container is running on localhost if containertype == "lxc": self.containerIP=self.calculateLocalContainerIP(self.cid) self.containerPort="22" if containertype == "docker": self.containerIP=self.calculateLocalContainerIP(1) self.containerPort=str(2000+int(self.cid)) self.config = Config.LoadConfig() self.SSHContainerPath = self.config['lbs']['SSHContainerPath'] self.logger = logger self.shell = Shell(logger) # we are reusing the slots, for caches etc self.slot = containername self.distro = "" self.release = "" self.arch = "" self.staticIP = "" self.packageSrcPath = packageSrcPath def calculateLocalContainerIP(self, cid): # test if we are inside a container as well # we just test if the host server for the build container is actually hosting the LBS application as well s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # just need to connect to any external host to know which is the IP address of the machine that hosts LBS s.connect((self.hostname, 80)) lbsipaddress=s.getsockname()[0].split('.') lbsipaddress.pop() # on CentOS: /etc/libvirt/qemu/networks/default.xml 192.168.122 # on Fedora 27: /etc/libvirt/qemu/networks/default.xml 192.168.124 # on Ubuntu 16.04: /etc/default/lxc-net 10.0.3 if '.'.join(lbsipaddress) == "192.168.122" or '.'.join(lbsipaddress) == "192.168.124" or '.'.join(lbsipaddress) == "10.0.3": return '.'.join(lbsipaddress) + "." + str(cid) # we are running uwsgi and lxc/docker on one host if os.path.isfile("/etc/redhat-release"): file = open("/etc/redhat-release", 'r') version = file.read() if "Fedora" in version: return "192.168.124." + str(cid) if "CentOS" in version: return "192.168.122." + str(cid) elif os.path.isfile("/etc/lsb-release"): file = open("/etc/lsb-release", 'r') version = file.read() if "Ubuntu" in version: return "10.0.3." + str(cid) def executeOnHost(self, command): if self.shell.executeshell('ssh -f -o "StrictHostKeyChecking no" -p ' + self.port + ' -i ' + self.SSHContainerPath + "/container_rsa root@" + self.hostname + " \"export LC_ALL=C; (" + command + ") 2>&1; echo \$?\""): return self.logger.getLastLine() == "0" return False def createmachine(self, distro, release, arch, staticIP): # not implemented here return False def startmachine(self): # not implemented here return False def executeInContainer(self, command): """Execute a command in a container via SSH""" # not implemented here return False def destroy(self): # not implemented here return False def stop(self): # not implemented here return False def rsyncContainerPut(self, src, dest): # not implemented here return False def rsyncContainerGet(self, path, dest = None): # not implemented here return False def rsyncHostPut(self, src, dest = None): # not implemented here return False def rsyncHostGet(self, path, dest = None): # not implemented here return False def installmount(self, localpath, hostpath = None): # not implemented here return False
def getPackagingInstructionsInternal(self, userconfig, username, projectname, branchname, gitprojectname, lbsproject, pathSrc): os.makedirs(pathSrc, exist_ok=True) needToDownload = True #we want a clean clone #but do not delete the tree if it is being used by another build t = None if os.path.isfile(pathSrc + 'lbs-' + projectname + '-lastused'): t = os.path.getmtime(pathSrc + 'lbs-' + projectname + '-lastused') # delete the tree only if it has not been used within the last 3 minutes if (time.time() - t) < 3 * 60: needToDownload = False # update the timestamp os.utime(pathSrc + 'lbs-' + projectname + '-lastused') else: open(pathSrc + 'lbs-' + projectname + '-lastused', 'a').close() headers = {} if not 'GitType' in userconfig or userconfig['GitType'] == 'github': url = lbsproject + "/archive/" + branchname + ".tar.gz" elif userconfig['GitType'] == 'gitlab': url = lbsproject + "/repository/archive.tar.gz?ref=" + branchname tokenfilename = self.config["lbs"][ "SSHContainerPath"] + "/" + username + "/" + projectname + "/gitlab_token" if os.path.isfile(tokenfilename): with open(tokenfilename, "r") as myfile: headers['PRIVATE-TOKEN'] = myfile.read().strip() # check if the version we have is still uptodate etagFile = pathSrc + 'lbs-' + projectname + '-etag' if needToDownload and os.path.isfile(etagFile): with open(etagFile, 'r') as content_file: Etag = content_file.read() headers['If-None-Match'] = Etag r = requests.get(url, headers=headers) if 'Etag' in r.headers and r.headers['Etag'] == '"' + Etag + '"': needToDownload = False if not needToDownload and os.path.isdir(pathSrc + 'lbs-' + projectname): # we can reuse the existing source, it was used just recently, or has not changed on the server self.StorePackageHashes(pathSrc + 'lbs-' + projectname, username, projectname, branchname) return # delete the working tree if os.path.isdir(pathSrc + 'lbs-' + projectname): shutil.rmtree(pathSrc + 'lbs-' + projectname) sourceFile = pathSrc + "/" + branchname + ".tar.gz" if os.path.isfile(sourceFile): os.remove(sourceFile) r = requests.get(url, headers=headers) if r.status_code == 401: raise Exception( "problem downloading the repository, access denied") elif not r.status_code == 200: raise Exception("problem downloading the repository " + url + ", HTTP error code " + str(r.status_code)) chunk_size = 100000 with open(sourceFile, 'wb') as fd: for chunk in r.iter_content(chunk_size): fd.write(chunk) if 'Etag' in r.headers: Etag = r.headers['Etag'] with open(etagFile, 'w') as fd: fd.write(Etag.strip('"')) shell = Shell(Logger()) if not 'GitType' in userconfig or userconfig['GitType'] == 'github': cmd = "cd " + pathSrc + ";" cmd += "tar xzf " + branchname + ".tar.gz; mv lbs-" + gitprojectname + "-" + branchname + " lbs-" + projectname shell.executeshell(cmd) elif userconfig['GitType'] == 'gitlab': cmd = "cd " + pathSrc + ";" cmd += "tar xzf " + branchname + ".tar.gz; mv lbs-" + gitprojectname + "-" + branchname + "-* lbs-" + projectname shell.executeshell(cmd) if os.path.isfile(sourceFile): os.remove(sourceFile) if not os.path.isdir(pathSrc + 'lbs-' + projectname): raise Exception("Problem with cloning the git repo") self.StorePackageHashes(pathSrc + 'lbs-' + projectname, username, projectname, branchname)
class Build: 'run one specific build of one package' def __init__(self, LBS, logger): self.LBS = LBS self.logger = logger self.container = None self.finished = False self.buildmachine = None self.config = Config.LoadConfig() def createbuildmachine(self, lxcdistro, lxcrelease, lxcarch, buildmachine, packageSrcPath): # create a container on a remote machine self.buildmachine = buildmachine con = Database(self.config) stmt = "SELECT * FROM machine WHERE name = ?" cursor = con.execute(stmt, (buildmachine,)) machine = cursor.fetchone() con.close() if machine['type'] == 'lxc': self.container = LXCContainer(buildmachine, machine, self.logger, packageSrcPath) elif machine['type'] == 'docker': self.container = DockerContainer(buildmachine, machine, self.logger, packageSrcPath) elif machine['type'] == 'copr': self.container = CoprContainer(buildmachine, machine, self.logger, packageSrcPath) return self.container.createmachine(lxcdistro, lxcrelease, lxcarch, buildmachine) def buildpackageOnCopr(self, username, projectname, packagename, branchname, packageSrcPath, lxcdistro, lxcrelease, lxcarch): # connect to copr coprtoken_filename = self.config['lbs']['SSHContainerPath'] + '/' + username + '/' + projectname + '/copr' if not os.path.isfile(coprtoken_filename): raise Exception("please download a token file from copr and save in " + coprtoken_filename) userconfig = self.config['lbs']['Users'][username] copr_projectname = projectname if 'CoprProjectName' in userconfig['Projects'][projectname]: copr_projectname = userconfig['Projects'][projectname]['CoprProjectName'] if not self.container.connectToCopr(coprtoken_filename, copr_projectname): raise Exception("problem connecting to copr, does the project " + copr_projectname + " already exist?") # calculate the release number release = self.container.getLatestReleaseFromCopr(packagename) if release is not None: if release.find('.') > -1: releasenumber = int(release[:release.find('.')]) afterreleasenumber = release[release.find('.'):] release = str(releasenumber+1)+afterreleasenumber else: release = str(int(release)+1) # build the src rpm locally, and move to public directory # simplification: tarball must be in the git repository # simplification: lbs must run on Fedora self.shell = Shell(self.logger) rpmbuildpath = "/run/uwsgi/rpmbuild_" + username + "_" + projectname + "_" + packagename self.shell.executeshell("mkdir -p " + rpmbuildpath + "/SOURCES; mkdir -p " + rpmbuildpath + "/SPECS") self.shell.executeshell("cp -R " + packageSrcPath + "/* " + rpmbuildpath + "/SOURCES; mv " + rpmbuildpath + "/SOURCES/*.spec " + rpmbuildpath + "/SPECS") if release is not None: self.shell.executeshell("sed -i 's/^Release:.*/Release: " + release + "/g' " + rpmbuildpath + "/SPECS/*.spec") if not self.shell.executeshell("rpmbuild --define '_topdir " + rpmbuildpath + "' -bs " + rpmbuildpath + "/SPECS/" + packagename + ".spec"): raise Exception("Problem with building the source rpm file for package " + packagename) myPath = username + "/" + projectname if 'Secret' in self.config['lbs']['Users'][username]: raise Exception("You cannot use a secret path when you are working with Copr") repoPath=self.config['lbs']['ReposPath'] + "/" + myPath + "/" + lxcdistro + "/" + lxcrelease + "/src" files = os.listdir(rpmbuildpath + "/SRPMS") if files is not None and len(files) == 1: srcrpmfilename = files[0] else: raise Exception("cannot find the source rpm, no files in " + rpmbuildpath + "/SRPMS") if not os.path.isfile(rpmbuildpath + "/SRPMS/" + srcrpmfilename): raise Exception("cannot find the source rpm, " + rpmbuildpath + "/SRPMS/" + srcrpmfilename + " is not a file") if not self.shell.executeshell("mkdir -p " + repoPath + " && mv " + rpmbuildpath + "/SRPMS/" + srcrpmfilename + " " + repoPath + " && rm -Rf " + rpmbuildpath): raise Exception("Problem moving the source rpm file") # tell copr to build this srpm. raise an exception if the build failed. if not self.container.buildProject(self.config['lbs']['DownloadUrl'] + "/repos/" + myPath + "/" + lxcdistro + "/" + lxcrelease + "/src/" + srcrpmfilename): raise Exception("problem building the package on copr") def buildpackageOnContainer(self, username, projectname, packagename, branchname, lxcdistro, lxcrelease, pathSrc): # install a mount for the project repo myPath = username + "/" + projectname if 'Secret' in self.config['lbs']['Users'][username]: myPath = username + "/" + self.config['lbs']['Users'][username]['Secret'] + "/" + projectname mountPath=self.config['lbs']['ReposPath'] + "/" + myPath + "/" + lxcdistro + "/" + lxcrelease if not self.container.installmount(mountPath, "/mnt" + mountPath, "/root/repo"): raise Exception("Problem with installmount") mountPath=self.config['lbs']['TarballsPath'] + "/" + myPath if not self.container.installmount(mountPath, "/mnt" + mountPath, "/root/tarball"): raise Exception("Problem with installmount") # prepare container, install packages that the build requires; this is specific to the distro self.buildHelper = BuildHelperFactory.GetBuildHelper(lxcdistro, self.container, username, projectname, packagename, branchname) if not self.buildHelper.PrepareMachineBeforeStart(): raise Exception("Problem with PrepareMachineBeforeStart") if self.container.startmachine(): self.logger.print("container has been started successfully") else: raise Exception("Problem with startmachine") if not self.buildHelper.PrepareMachineAfterStart(): raise Exception("Problem with PrepareMachineAfterStart") if not self.buildHelper.PrepareForBuilding(): raise Exception("Problem with PrepareForBuilding") # copy the repo to the container self.container.rsyncContainerPut(pathSrc+'lbs-'+projectname, "/root/lbs-"+projectname) # copy the keys to the container sshContainerPath = self.config['lbs']['SSHContainerPath'] if os.path.exists(sshContainerPath + '/' + username + '/' + projectname): self.container.rsyncContainerPut(sshContainerPath + '/' + username + '/' + projectname + '/*', '/root/.ssh/') self.container.executeInContainer('chmod 600 /root/.ssh/*') if not self.buildHelper.DownloadSources(): raise Exception("Problem with DownloadSources") if not self.buildHelper.InstallRepositories(self.config['lbs']['DownloadUrl']): raise Exception("Problem with InstallRepositories") if not self.buildHelper.SetupEnvironment(branchname): raise Exception("Setup script did not succeed") if not self.buildHelper.InstallRequiredPackages(): raise Exception("Problem with InstallRequiredPackages") # disable the network, so that only code from the tarball is being used if not self.buildHelper.DisableOutgoingNetwork(): raise Exception("Problem with disabling the network") if not self.buildHelper.BuildPackage(): raise Exception("Problem with building the package") myPath = username + "/" + projectname if 'Secret' in self.config['lbs']['Users'][username]: myPath = username + "/" + self.config['lbs']['Users'][username]['Secret'] + "/" + projectname srcPath=self.config['lbs']['ReposPath'] + "/" + myPath + "/" + lxcdistro + "/" + lxcrelease destPath=srcPath[:srcPath.rindex("/")] srcPath="/mnt"+srcPath if not self.container.rsyncHostGet(srcPath, destPath): raise Exception("Problem with syncing repos") srcPath=self.config['lbs']['TarballsPath'] + "/" + myPath destPath=srcPath[:srcPath.rindex("/")] srcPath="/mnt"+srcPath if not self.container.rsyncHostGet(srcPath, destPath): raise Exception("Problem with syncing tarballs") # create repo file self.buildHelper.CreateRepoFile() def buildpackage(self, username, projectname, packagename, branchname, lxcdistro, lxcrelease, lxcarch, buildmachine, jobId): userconfig = self.config['lbs']['Users'][username] self.logger.startTimer() self.logger.print(" * Starting at " + strftime("%Y-%m-%d %H:%M:%S GMT%z")) self.logger.print(" * Preparing the machine...") # get the sources of the packaging instructions gotPackagingInstructions = False try: pathSrc=self.LBS.getPackagingInstructions(userconfig, username, projectname, branchname) packageSrcPath=pathSrc + '/lbs-'+projectname + '/' + packagename gotPackagingInstructions = True except Exception as e: print(e) self.logger.print("LBSERROR: "+str(e)+ "; for more details see /var/log/uwsgi.log") jobFailed = True if not gotPackagingInstructions: self.LBS.ReleaseMachine(buildmachine, jobFailed) elif self.createbuildmachine(lxcdistro, lxcrelease, lxcarch, buildmachine, packageSrcPath): try: if type(self.container) is CoprContainer: self.buildpackageOnCopr(username, projectname, packagename, branchname, packageSrcPath, lxcdistro, lxcrelease, lxcarch) else: self.buildpackageOnContainer(username, projectname, packagename, branchname, lxcdistro, lxcrelease, pathSrc) self.logger.print("Success!") self.LBS.MarkPackageAsBuilt(username, projectname, packagename, branchname, lxcdistro, lxcrelease, lxcarch) jobFailed = False except Exception as e: # TODO: logging to log file does not work yet? logging.basicConfig(level=logging.DEBUG, filename='/var/log/lbs.log') logging.exception("Error happened...") self.logger.print("LBSERROR: "+str(e)) finally: self.LBS.ReleaseMachine(buildmachine, jobFailed) else: self.logger.print("LBSERROR: There is a problem with creating the container!") self.LBS.ReleaseMachine(buildmachine, jobFailed) self.finished = True logpath=self.logger.getLogPath(username, projectname, packagename, branchname, lxcdistro, lxcrelease, lxcarch) buildnumber=self.logger.store(self.config['lbs']['DeleteLogAfterDays'], self.config['lbs']['KeepMinimumLogs'], logpath) if self.logger.hasLBSERROR() or not self.config['lbs']['SendEmailOnSuccess'] == False: if self.config['lbs']['EmailFromAddress'] == '*****@*****.**': self.logger.print("Please configure the email settings for sending notification emails") else: self.logger.email(self.config['lbs']['EmailFromAddress'], userconfig['EmailToAddress'], "LBS Result for " + projectname + "/" + packagename, self.config['lbs']['LBSUrl'] + "/logs/" + logpath + "/" + str(buildnumber)) # now mark the build finished con = Database(self.config) stmt = "UPDATE build SET status='FINISHED', finished=?, buildsuccess=?, buildnumber=? WHERE id = ?" lastBuild = Logger().getLastBuild(username, projectname, packagename, branchname, lxcdistro+"/"+lxcrelease+"/"+lxcarch) con.execute(stmt, (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), lastBuild["resultcode"], lastBuild["number"], jobId)) con.commit() con.close() self.logger.clean() return self.logger.get()
def getPackagingInstructionsInternal(self, userconfig, username, projectname, branchname, gitprojectname, lbsproject, pathSrc): os.makedirs(pathSrc, exist_ok=True) needToDownload = True #we want a clean clone #but do not delete the tree if it is being used by another build t = None if os.path.isfile(pathSrc+'lbs-'+projectname+'-lastused'): t = os.path.getmtime(pathSrc+'lbs-'+projectname+'-lastused') # delete the tree only if it has not been used within the last 3 minutes if (time.time() - t) < 3*60: needToDownload = False # update the timestamp os.utime(pathSrc+'lbs-'+projectname+'-lastused') else: open(pathSrc+'lbs-'+projectname+'-lastused', 'a').close() headers = {} if not 'GitType' in userconfig or userconfig['GitType'] == 'github': url=lbsproject + "/archive/" + branchname + ".tar.gz" elif userconfig['GitType'] == 'gitlab': url=lbsproject + "/repository/archive.tar.gz?ref=" + branchname tokenfilename=self.config["lbs"]["SSHContainerPath"] + "/" + username + "/" + projectname + "/gitlab_token" if os.path.isfile(tokenfilename): with open (tokenfilename, "r") as myfile: headers['PRIVATE-TOKEN'] = myfile.read().strip() # check if the version we have is still uptodate etagFile = pathSrc+'lbs-'+projectname+'-etag' if needToDownload and os.path.isfile(etagFile): with open(etagFile, 'r') as content_file: Etag = content_file.read() headers['If-None-Match'] = Etag r = requests.get(url, headers=headers) if 'Etag' in r.headers and r.headers['Etag'] == '"' + Etag + '"': needToDownload = False if not needToDownload and os.path.isdir(pathSrc+'lbs-'+projectname): # we can reuse the existing source, it was used just recently, or has not changed on the server self.StorePackageHashes(pathSrc+'lbs-'+projectname, username, projectname, branchname) return # delete the working tree if os.path.isdir(pathSrc+'lbs-'+projectname): shutil.rmtree(pathSrc+'lbs-'+projectname) sourceFile = pathSrc + "/" + branchname + ".tar.gz" if os.path.isfile(sourceFile): os.remove(sourceFile) r = requests.get(url, headers=headers) if r.status_code == 401: raise Exception("problem downloading the repository, access denied") elif not r.status_code == 200: raise Exception("problem downloading the repository " + url + ", HTTP error code " + str(r.status_code)) chunk_size = 100000 with open(sourceFile, 'wb') as fd: for chunk in r.iter_content(chunk_size): fd.write(chunk) if 'Etag' in r.headers: Etag = r.headers['Etag'] with open(etagFile, 'w') as fd: fd.write(Etag.strip('"')) shell = Shell(Logger()) if not 'GitType' in userconfig or userconfig['GitType'] == 'github': cmd="cd " + pathSrc + ";" cmd+="tar xzf " + branchname + ".tar.gz; mv lbs-" + gitprojectname + "-" + branchname + " lbs-" + projectname shell.executeshell(cmd) elif userconfig['GitType'] == 'gitlab': cmd="cd " + pathSrc + ";" cmd+="tar xzf " + branchname + ".tar.gz; mv lbs-" + gitprojectname + "-" + branchname + "-* lbs-" + projectname shell.executeshell(cmd) if os.path.isfile(sourceFile): os.remove(sourceFile) if not os.path.isdir(pathSrc+'lbs-'+projectname): raise Exception("Problem with cloning the git repo") self.StorePackageHashes(pathSrc+'lbs-'+projectname, username, projectname, branchname)
class RemoteContainer: def __init__(self, containername, configBuildMachine, logger, packageSrcPath, containertype): self.hostname = containername self.containertype = containertype self.staticMachine = (True if ('static' in configBuildMachine and configBuildMachine['static'] == "t") else False) self.port = "22" if configBuildMachine['port'] is not None: self.port = str(configBuildMachine['port']) self.cid = 10 if configBuildMachine['cid'] is not None: self.cid = configBuildMachine['cid'] self.containername = str(self.cid).zfill(3) + "-" + containername if containertype == "lxd": self.containername = "l" + str( self.cid).zfill(3) + "-" + containername.replace(".", "-") self.containerIP = socket.gethostbyname(self.hostname) self.containerPort = str(2000 + int(self.cid)) if configBuildMachine['local'] is not None and configBuildMachine[ 'local'] == "t": # the host server for the build container is actually hosting the LBS application as well # or the container is running on localhost if containertype == "lxc": self.containerIP = self.calculateLocalContainerIP(self.cid) self.containerPort = "22" if containertype == "lxd": self.containerIP = self.calculateLocalContainerIP(self.cid) self.containerPort = "22" if containertype == "docker": self.containerIP = self.calculateLocalContainerIP(1) self.containerPort = str(2000 + int(self.cid)) self.config = Config.LoadConfig() self.SSHContainerPath = self.config['lbs']['SSHContainerPath'] self.logger = logger self.shell = Shell(logger) # we are reusing the slots, for caches etc self.slot = containername self.distro = "" self.release = "" self.arch = "" self.staticIP = "" self.packageSrcPath = packageSrcPath def calculateLocalContainerIP(self, cid): # for LXD, we always configure the bridge with 10.0.4: # lxc network create lxdbr0 ipv6.address=none ipv4.address=10.0.4.1/24 ipv4.nat=true if self.containertype == "lxd": return "10.0.4." + str(cid) # test if we are inside a container as well # we just test if the host server for the build container is actually hosting the LBS application as well s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # just need to connect to any external host to know which is the IP address of the machine that hosts LBS s.connect((self.hostname, 80)) lbsipaddress = s.getsockname()[0].split('.') lbsipaddress.pop() # on CentOS: /etc/libvirt/qemu/networks/default.xml 192.168.122 # on Fedora 27: /etc/libvirt/qemu/networks/default.xml 192.168.124 # on Ubuntu 16.04: /etc/default/lxc-net 10.0.3 if '.'.join(lbsipaddress) == "192.168.122" or '.'.join( lbsipaddress) == "192.168.124" or '.'.join( lbsipaddress) == "10.0.3": return '.'.join(lbsipaddress) + "." + str(cid) # we are running uwsgi and lxc/docker on one host if os.path.isfile("/etc/redhat-release"): file = open("/etc/redhat-release", 'r') version = file.read() if "Fedora" in version: return "192.168.124." + str(cid) if "CentOS" in version: return "192.168.122." + str(cid) elif os.path.isfile("/etc/lsb-release"): file = open("/etc/lsb-release", 'r') version = file.read() if "Ubuntu" in version: return "10.0.3." + str(cid) def executeOnHost(self, command): if self.shell.executeshell('ssh -f -o "StrictHostKeyChecking no" -p ' + self.port + ' -i ' + self.SSHContainerPath + "/container_rsa root@" + self.hostname + " \"export LC_ALL=C; (" + command + ") 2>&1; echo \$?\""): return self.logger.getLastLine() == "0" return False def createmachine(self, distro, release, arch, staticIP): # not implemented here return False def startmachine(self): # not implemented here return False def executeInContainer(self, command): """Execute a command in a container via SSH""" # not implemented here return False def destroy(self): # not implemented here return False def stop(self): # not implemented here return False def rsyncContainerPut(self, src, dest): # not implemented here return False def rsyncContainerGet(self, path, dest=None): # not implemented here return False def rsyncHostPut(self, src, dest=None): # not implemented here return False def rsyncHostGet(self, path, dest=None): # not implemented here return False def installmount(self, localpath, hostpath=None): # not implemented here return False
class Build: 'run one specific build of one package' def __init__(self, LBS, logger): self.LBS = LBS self.logger = logger self.container = None self.finished = False self.buildmachine = None self.config = Config.LoadConfig() def createbuildmachine(self, lxcdistro, lxcrelease, lxcarch, buildmachine, packageSrcPath): # create a container on a remote machine self.buildmachine = buildmachine con = Database(self.config) stmt = "SELECT * FROM machine WHERE name = ?" cursor = con.execute(stmt, (buildmachine, )) machine = cursor.fetchone() con.close() if machine['type'] == 'lxc': self.container = LXCContainer(buildmachine, machine, self.logger, packageSrcPath) elif machine['type'] == 'lxd': self.container = LXDContainer(buildmachine, machine, self.logger, packageSrcPath) elif machine['type'] == 'docker': self.container = DockerContainer(buildmachine, machine, self.logger, packageSrcPath) elif machine['type'] == 'copr': self.container = CoprContainer(buildmachine, machine, self.logger, packageSrcPath) return self.container.createmachine(lxcdistro, lxcrelease, lxcarch, buildmachine) def buildpackageOnCopr(self, username, projectname, packagename, branchname, packageSrcPath, lxcdistro, lxcrelease, lxcarch): # connect to copr coprtoken_filename = self.config['lbs'][ 'SSHContainerPath'] + '/' + username + '/' + projectname + '/copr' if not os.path.isfile(coprtoken_filename): raise Exception( "please download a token file from copr and save in " + coprtoken_filename) userconfig = self.config['lbs']['Users'][username] copr_projectname = projectname if 'CoprProjectName' in userconfig['Projects'][projectname]: copr_projectname = userconfig['Projects'][projectname][ 'CoprProjectName'] copr_username = username if 'CoprUserName' in userconfig['Projects'][projectname]: copr_username = userconfig['Projects'][projectname]['CoprUserName'] if not self.container.connectToCopr(coprtoken_filename, copr_username, copr_projectname): raise Exception("problem connecting to copr, does the project " + copr_projectname + " already exist?") # calculate the release number release = self.container.getLatestReleaseFromCopr(packagename) if release is not None: if release.find('.') > -1: releasenumber = int(release[:release.find('.')]) afterreleasenumber = release[release.find('.'):] release = str(releasenumber + 1) + afterreleasenumber else: release = str(int(release) + 1) # build the src rpm locally, and move to public directory # simplification: tarball must be in the git repository self.shell = Shell(self.logger) rpmbuildpath = "/run/uwsgi/rpmbuild_" + username + "_" + projectname + "_" + packagename self.shell.executeshell("mkdir -p " + rpmbuildpath + "/SOURCES; mkdir -p " + rpmbuildpath + "/SPECS") self.shell.executeshell("cp -R " + packageSrcPath + "/* " + rpmbuildpath + "/SOURCES; mv " + rpmbuildpath + "/SOURCES/*.spec " + rpmbuildpath + "/SPECS") if release is not None: self.shell.executeshell("sed -i 's/^Release:.*/Release: " + release + "/g' " + rpmbuildpath + "/SPECS/*.spec") if not self.shell.executeshell("rpmbuild --define '_topdir " + rpmbuildpath + "' -bs " + rpmbuildpath + "/SPECS/" + packagename + ".spec"): raise Exception( "Problem with building the source rpm file for package " + packagename) myPath = username + "/" + projectname if 'Secret' in self.config['lbs']['Users'][username]: raise Exception( "You cannot use a secret path when you are working with Copr") repoPath = self.config['lbs'][ 'ReposPath'] + "/" + myPath + "/" + lxcdistro + "/" + lxcrelease + "/src" files = os.listdir(rpmbuildpath + "/SRPMS") if files is not None and len(files) == 1: srcrpmfilename = files[0] else: raise Exception("cannot find the source rpm, no files in " + rpmbuildpath + "/SRPMS") if not os.path.isfile(rpmbuildpath + "/SRPMS/" + srcrpmfilename): raise Exception("cannot find the source rpm, " + rpmbuildpath + "/SRPMS/" + srcrpmfilename + " is not a file") if not self.shell.executeshell("mkdir -p " + repoPath + " && mv " + rpmbuildpath + "/SRPMS/" + srcrpmfilename + " " + repoPath + " && rm -Rf " + rpmbuildpath): raise Exception("Problem moving the source rpm file") # tell copr to build this srpm. raise an exception if the build failed. if not self.container.buildProject(self.config['lbs']['DownloadUrl'] + "/repos/" + myPath + "/" + lxcdistro + "/" + lxcrelease + "/src/" + srcrpmfilename): raise Exception("problem building the package on copr") def buildpackageOnContainer(self, username, projectname, packagename, branchname, lxcdistro, lxcrelease, pathSrc): # install a mount for the project repo myPath = username + "/" + projectname if 'Secret' in self.config['lbs']['Users'][username]: myPath = username + "/" + self.config['lbs']['Users'][username][ 'Secret'] + "/" + projectname mountPath = self.config['lbs'][ 'ReposPath'] + "/" + myPath + "/" + lxcdistro + "/" + lxcrelease if not self.container.installmount(mountPath, "/mnt" + mountPath, "/root/repo"): raise Exception("Problem with installmount") mountPath = self.config['lbs']['TarballsPath'] + "/" + myPath if not self.container.installmount(mountPath, "/mnt" + mountPath, "/root/tarball"): raise Exception("Problem with installmount") # prepare container, install packages that the build requires; this is specific to the distro self.buildHelper = BuildHelperFactory.GetBuildHelper( lxcdistro, self.container, username, projectname, packagename, branchname) if not self.buildHelper.PrepareMachineBeforeStart(): raise Exception("Problem with PrepareMachineBeforeStart") if self.container.startmachine(): self.logger.print("container has been started successfully") else: raise Exception("Problem with startmachine") if not self.buildHelper.PrepareMachineAfterStart(): raise Exception("Problem with PrepareMachineAfterStart") if not self.buildHelper.PrepareForBuilding(): raise Exception("Problem with PrepareForBuilding") # copy the repo to the container self.container.rsyncContainerPut(pathSrc + 'lbs-' + projectname, "/root/lbs-" + projectname) # copy the keys to the container sshContainerPath = self.config['lbs']['SSHContainerPath'] if os.path.exists(sshContainerPath + '/' + username + '/' + projectname): self.container.rsyncContainerPut( sshContainerPath + '/' + username + '/' + projectname + '/*', '/root/.ssh/') self.container.executeInContainer('chmod 600 /root/.ssh/*') if not self.buildHelper.DownloadSources(): raise Exception("Problem with DownloadSources") if not self.buildHelper.InstallRepositories( self.config['lbs']['DownloadUrl']): raise Exception("Problem with InstallRepositories") if not self.buildHelper.SetupEnvironment(branchname): raise Exception("Setup script did not succeed") if not self.buildHelper.InstallRequiredPackages(): raise Exception("Problem with InstallRequiredPackages") # disable the network, so that only code from the tarball is being used if not self.buildHelper.DisableOutgoingNetwork(): raise Exception("Problem with disabling the network") if not self.buildHelper.BuildPackage(): raise Exception("Problem with building the package") myPath = username + "/" + projectname if 'Secret' in self.config['lbs']['Users'][username]: myPath = username + "/" + self.config['lbs']['Users'][username][ 'Secret'] + "/" + projectname srcPath = self.config['lbs'][ 'ReposPath'] + "/" + myPath + "/" + lxcdistro + "/" + lxcrelease destPath = srcPath[:srcPath.rindex("/")] srcPath = "/mnt" + srcPath if not self.container.rsyncHostGet(srcPath, destPath): raise Exception("Problem with syncing repos") srcPath = self.config['lbs']['TarballsPath'] + "/" + myPath destPath = srcPath[:srcPath.rindex("/")] srcPath = "/mnt" + srcPath if not self.container.rsyncHostGet(srcPath, destPath): raise Exception("Problem with syncing tarballs") # create repo file self.buildHelper.CreateRepoFile() def buildpackage(self, username, projectname, packagename, branchname, lxcdistro, lxcrelease, lxcarch, buildmachine, jobId): userconfig = self.config['lbs']['Users'][username] self.logger.startTimer() self.logger.print(" * Starting at " + strftime("%Y-%m-%d %H:%M:%S GMT%z")) self.logger.print(" * Preparing the machine...") # get the sources of the packaging instructions gotPackagingInstructions = False try: pathSrc = self.LBS.getPackagingInstructions( userconfig, username, projectname, branchname) packageSrcPath = pathSrc + '/lbs-' + projectname + '/' + packagename gotPackagingInstructions = True except Exception as e: print(e) self.logger.print("LBSERROR: " + str(e) + "; for more details see /var/log/uwsgi.log") jobFailed = True if not gotPackagingInstructions: self.LBS.ReleaseMachine(buildmachine, jobFailed) elif self.createbuildmachine(lxcdistro, lxcrelease, lxcarch, buildmachine, packageSrcPath): try: if type(self.container) is CoprContainer: self.buildpackageOnCopr(username, projectname, packagename, branchname, packageSrcPath, lxcdistro, lxcrelease, lxcarch) else: self.buildpackageOnContainer(username, projectname, packagename, branchname, lxcdistro, lxcrelease, pathSrc) self.logger.print("Success!") self.LBS.MarkPackageAsBuilt(username, projectname, packagename, branchname, lxcdistro, lxcrelease, lxcarch) jobFailed = False except Exception as e: self.logger.print("LBSERROR: " + str(e), 0) finally: self.LBS.ReleaseMachine(buildmachine, jobFailed) else: self.logger.print( "LBSERROR: There is a problem with creating the container!") self.LBS.ReleaseMachine(buildmachine, jobFailed) self.finished = True logpath = self.logger.getLogPath(username, projectname, packagename, branchname, lxcdistro, lxcrelease, lxcarch) buildnumber = self.logger.store( self.config['lbs']['DeleteLogAfterDays'], self.config['lbs']['KeepMinimumLogs'], logpath) if self.logger.hasLBSERROR( ) or not self.config['lbs']['SendEmailOnSuccess'] == False: if self.config['lbs']['EmailFromAddress'] == '*****@*****.**': self.logger.print( "Please configure the email settings for sending notification emails" ) else: try: self.logger.email( self.config['lbs']['EmailFromAddress'], userconfig['EmailToAddress'], "LBS Result for " + projectname + "/" + packagename, self.config['lbs']['LBSUrl'] + "/logs/" + logpath + "/" + str(buildnumber)) except Exception as e: self.logger.print("ERROR: we could not send the email") # now mark the build finished con = Database(self.config) stmt = "UPDATE build SET status='FINISHED', finished=?, buildsuccess=?, buildnumber=? WHERE id = ?" lastBuild = Logger().getLastBuild( username, projectname, packagename, branchname, lxcdistro + "/" + lxcrelease + "/" + lxcarch) con.execute(stmt, (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), lastBuild["resultcode"], lastBuild["number"], jobId)) con.commit() con.close() self.logger.clean() return self.logger.get()