def local_repo(self, distribution, files): archive_conf_data = """Dir { ArchiveDir "%s/%s"; }; BinDirectory "dists/unstable" { Packages "dists/unstable/main/binary-amd64/Packages"; SrcPackages "dists/unstable/main/source/Sources"; }; BinDirectory "dists/stable" { Packages "dists/stable/main/binary-amd64/Packages"; SrcPackages "dists/stable/main/source/Sources"; }; BinDirectory "dists/testing" { Packages "dists/testing/main/binary-amd64/Packages"; SrcPackages "dists/testing/main/source/Sources"; }; BinDirectory "dists/experimental" { Packages "dists/experimental/main/binary-amd64/Packages"; SrcPackages "dists/experimental/main/source/Sources"; };""" repo_bin_path = os.path.join(BrickConfig().get("local_repo", "dir"), self.project.group_name, 'dists/%s/main/binary-amd64/' % distribution) repo_src_path = os.path.join(BrickConfig().get("local_repo", "dir"), self.project.group_name, 'dists/%s/main/source/' % distribution) archive_conf_file = os.path.join( BrickConfig().get("local_repo", "dir"), self.project.group_name, 'archive.conf') if not os.path.isdir(repo_bin_path) or not os.path.isdir(repo_src_path): os.makedirs(repo_bin_path) os.makedirs(repo_src_path) if not os.path.isfile(archive_conf_file): with open(archive_conf_file, 'w') as tmpfh: tmpfh.write( archive_conf_data % ( BrickConfig().get("local_repo", "dir"), self.project.group_name )) workspace = BrickConfig().get("workspace", "dir") for f in files: f = f.strip() if f.endswith('.dsc') or f.endswith('.tar.gz'): shutil.copy(os.path.join(workspace, f), os.path.join(repo_src_path, f)) elif f.endswith('.deb'): shutil.copy(os.path.join(workspace, f), os.path.join(repo_bin_path, f)) repo_base_path = os.path.join(BrickConfig().get('local_repo', 'dir'), self.project.group_name)
def upload_files(self, distribution, files): repository_url, user, passwd = self.project.repository() if not repository_url: return 0 # os.chdir(BrickConfig().get('workspace', 'dir')) workspace = BrickConfig().get('workspace', 'dir') ftp = ftplib.FTP(repository_url, user, passwd) try: ftp.cwd(distribution) for f in files: log.info("\t%s: " % os.path.join(workspace, f)) with open(os.path.join(workspace, f), 'rb') as tmpfh: ftp.storbinary("STOR %s" % f, tmpfh) log.info("done.") except Exception, e: log.info(repr(e))
from utils import pystache from twisted.internet import threads, reactor, defer from config import BrickConfig from model.project import Project # from builder_rpm import BuilderRpm from builder_deb import BuilderDeb from build_options import BuildOptions from build_container import BuildContainer from current_build import CurrentBuild #from dreque import Dreque config = BrickConfig() redis_server = config.get('redis', 'redis-server') log_file = config.get('log', 'file') #queue = Dreque(redis_server) logging.basicConfig(filename=log_file, level=logging.DEBUG) log = logging.getLogger('builder') @defer.inlineCallbacks def build_project(kargs): builder = Builder(kargs['project']) kargs.pop('project') yield builder.build_project(**kargs)
from bricklayer.utils import pystache import git from twisted.internet import threads, reactor, defer from config import BrickConfig from projects import Projects from builder_rpm import BuilderRpm from builder_deb import BuilderDeb from build_options import BuildOptions from build_container import BuildContainer #from dreque import Dreque config = BrickConfig() redis_server = config.get('redis', 'redis-server') log_file = config.get('log', 'file') #queue = Dreque(redis_server) logging.basicConfig(filename=log_file, level=logging.DEBUG) log = logging.getLogger('builder') @defer.inlineCallbacks def build_project(kargs): builder = Builder(kargs['project']) kargs.pop('project') yield builder.build_project(**kargs) class Builder(object):
def build(self, branch, last_tag=None, force_version=None, force_release=None): templates = {} templates_dir = os.path.join(self.builder.templates_dir, 'deb') control_data_new = None self.build_info = BuildInfo(self.project.name) logfile = os.path.join( BrickConfig().get("workspace", "dir"), 'log', '%s.%s.log' % (self.project.name, self.build_info.build_id)) log.info("build log file: %s" % logfile) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout self.debian_dir = os.path.join(self.builder.workdir, 'debian') if last_tag is not None: os.environ.update({'BRICKLAYER_RELEASE': last_tag.split('_')[0]}) os.environ.update({'BRICKLAYER_TAG': last_tag}) # Not now #self.build_install_deps() if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd: self.project.install_cmd = 'cp -r \`ls | grep -v debian\` debian/tmp/%s' % ( self.project.install_prefix) changelog_entry, changelog_data = self.configure_changelog(branch) if changelog_entry and changelog_data: if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) changelog_data.update({ 'version': self.project.version('stable'), 'branch': 'stable' }) self.build_info.version(self.project.version('stable')) self.build_info.release('stable') elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) changelog_data.update({ 'version': self.project.version('testing'), 'branch': 'testing' }) self.build_info.version(self.project.version('testing')) self.build_info.release('testing') elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) changelog_data.update({ 'version': self.project.version('unstable'), 'branch': 'unstable' }) self.build_info.version(self.project.version('unstable')) self.build_info.release('unstable') else: """ otherwise it should change the distribution to experimental """ version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str( int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) changelog_data.update({ 'version': self.project.version(branch), 'branch': 'experimental' }) self.build_info.version(self.project.version(branch)) self.build_info.release('experimental:%s' % branch) with file( os.path.join(self.builder.workdir, 'debian', 'changelog'), 'w') as fh: fh.write(changelog_entry % changelog_data) else: self.build_info.version(force_version) self.build_info.release(force_release) self.project.version(branch, force_version) self.project.version(force_release, force_version) rvm_env = {} rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: with open(rvm_rc) as tmpfh: rvmexec = tmpfh.read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/rvm/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: os_env = dict(os.environ) for k in ("PATH", "GEM_HOME", "BUNDLER_PATH"): if (k in os_env): del (os_env[k]) rvm_env.update(os_env) log.info(rvm_env) os.chmod(os.path.join(self.debian_dir, 'rules'), stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH) dpkg_cmd = self.builder._exec([ 'dpkg-buildpackage', '-rfakeroot', '-tc', '-k%s' % BrickConfig().get('gpg', 'keyid') ], cwd=self.builder.workdir, env=rvm_env, stdout=self.stdout, stderr=self.stderr, close_fds=True) dpkg_cmd.wait() clean_cmd = self.builder._exec(['dh', 'clean'], cwd=self.builder.workdir, stdout=self.stdout, stderr=self.stderr, close_fds=True) clean_cmd.wait()
sys.path.append(os.path.dirname(__file__)) from projects import Projects from groups import Groups from builder import Builder from build_info import BuildInfo from config import BrickConfig import cyclone.web import cyclone.escape from twisted.internet import reactor from twisted.python import log from dreque import Dreque brickconfig = BrickConfig() queue = Dreque(brickconfig.get('redis', 'redis-server')) class Project(cyclone.web.RequestHandler): def post(self, *args): if len(args) >= 1: name = args[0] project = Projects(name) for key, value in self.request.arguments.iteritems(): if key in ("git_url", "version", "build_cmd", "install_cmd"): setattr(project, key, value[0]) project.save() try: if not Projects(self.get_argument('name')).exists():
sys.path.append(os.path.dirname(__file__)) from projects import Projects from groups import Groups from git import Git from builder import Builder from build_info import BuildInfo from config import BrickConfig import cyclone.web import cyclone.escape from twisted.internet import reactor from twisted.python import log from twisted.application import service, internet from dreque import Dreque brickconfig = BrickConfig() queue = Dreque(brickconfig.get('redis', 'redis-server')) class Project(cyclone.web.RequestHandler): def post(self, *args): if len(args) >= 1: name = args[0] project = Projects(name) for key, value in self.request.arguments.iteritems(): if key in ("git_url", "version", "build_cmd", "install_cmd"): setattr(project, key, value[0]) project.save() try:
sys.path.append(os.path.dirname(__file__)) from projects import Projects from groups import Groups from git import Git from builder import Builder, build_project from build_info import BuildInfo from current_build import CurrentBuild from config import BrickConfig import cyclone.web import cyclone.escape from twisted.internet import reactor from twisted.python import log from twisted.application import service, internet brickconfig = BrickConfig() class Project(cyclone.web.RequestHandler): def post(self, *args): if len(args) >= 1: name = args[0] project = Projects(name) for key, value in self.request.arguments.iteritems(): if key in ("git_url", "version", "build_cmd", "install_cmd"): setattr(project, key, value[0]) project.save() try: if not Projects(self.get_argument('name')).exists(): raise
from config import BrickConfig print BrickConfig('./config/bricklayer.ini').get('databases', 'uri')
sys.path.append(os.path.dirname(__file__)) from projects import Projects from groups import Groups from git import Git from builder import Builder, build_project from build_info import BuildInfo from current_build import CurrentBuild from config import BrickConfig import cyclone.web import cyclone.escape from twisted.internet import reactor from twisted.python import log from twisted.application import service, internet brickconfig = BrickConfig() class Project(cyclone.web.RequestHandler): def post(self, *args): if len(args) >= 1: name = args[0] project = Projects(name) for key, value in self.request.arguments.iteritems(): if key in ("git_url", "version", "build_cmd", "install_cmd"): setattr(project, key, value[0]) project.save() try: if not Projects(self.get_argument("name")).exists(): raise
from projects import Projects from groups import Groups from git import Git from builder import Builder from build_info import BuildInfo from config import BrickConfig import cyclone.web import cyclone.escape from twisted.internet import reactor from twisted.python import log from twisted.application import service, internet from dreque import Dreque brickconfig = BrickConfig() queue = Dreque(brickconfig.get('redis', 'redis-server')) class Project(cyclone.web.RequestHandler): def post(self, *args): if len(args) >= 1: name = args[0] project = Projects(name) for key, value in self.request.arguments.iteritems(): if key in ("git_url", "version", "build_cmd", "install_cmd"): setattr(project, key, value[0]) project.save() try: if not Projects(self.get_argument('name')).exists():
def draw_routes(): return Application([ (r'/static/(.*)', cyclone.web.StaticFileHandler, { 'path': brickconfig.get('static', 'dir') }), (r'/project', ProjectIndexController), (r'/project/?(.*)', ProjectController), (r'/group', GroupController), (r'/build/(.*)', BuildController), # (r'/build/current', Current), # (r'/group/?(.*)', Group), # (r'/branch/(.*)', Branch), # (r'/clear/(.*)', Clear), # (r'/log/(.*)/+(.*)', Log), # (r'/repo/(.*)', cyclone.web.StaticFileHandler, {'path': brickconfig.get('local_repo', 'dir')}), (r'/', MainController) ]) def start(application): server = internet.TCPServer(int(brickconfig.get('server', 'port')), draw_routes()) server.setServiceParent(application) brickconfig = BrickConfig() application = service.Application("bricklayer_rest") start(application)
class BuilderDeb(): def __init__(self, builder): self.builder = builder self.project = self.builder.project log.info("Debian builder initialized: %s" % self.builder.workdir) log.info("Building with options: %s" % self.builder.build_options.options) def build(self, branch, last_tag=None, force_version=None, force_release=None): templates = {} templates_dir = os.path.join(self.builder.templates_dir, 'deb') debian_dir = os.path.join(self.builder.workdir, 'debian') control_data_new = None self.build_info = BuildInfo(self.project.name) logfile = os.path.join( self.builder.workspace, 'log', '%s.%s.log' % ( self.project.name, self.build_info.build_id ) ) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd : self.project.install_cmd = 'cp -r \`ls | grep -v debian\` debian/tmp/%s' % ( self.project.install_prefix ) template_data = { 'name': self.project.name, 'version': "%s" % (self.project.version(branch)), 'build_cmd': self.project.build_cmd, 'install_cmd': self.builder.mod_install_cmd, 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } changelog = os.path.join(self.builder.workdir, 'debian', 'changelog') if hasattr(self.builder.build_options, 'changelog') and self.builder.build_options.changelog: if os.path.isfile(changelog): os.rename(changelog, "%s.save" % changelog) def read_file_data(f): template_fd = open(os.path.join(templates_dir, f)) templates[f] = pystache.template.Template(template_fd.read()).render(context=template_data) template_fd.close() if not os.path.isdir(debian_dir): map(read_file_data, ['changelog', 'control', 'rules']) os.makedirs( os.path.join( debian_dir, self.project.name, self.project.install_prefix)) for filename, data in templates.iteritems(): open(os.path.join(debian_dir, filename), 'w').write(data) changelog_entry = """%(name)s (%(version)s) %(branch)s; urgency=low * Latest commits %(commits)s -- %(username)s <%(email)s> %(date)s """ changelog_data = { 'name': self.project.name, 'version': self.project.version(branch), 'branch': branch, 'commits': ' '.join(self.builder.git.log()), 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('stable'), 'branch': 'stable'}) self.build_info.version(self.project.version('stable')) self.build_info.release('stable') elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('testing'), 'branch': 'testing'}) self.build_info.version(self.project.version('testing')) self.build_info.release('testing') elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('unstable'), 'branch': 'unstable'}) self.build_info.version(self.project.version('unstable')) self.build_info.release('unstable') else: """ otherwise it should change the distribution to experimental """ if self.project.version(branch): version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) changelog_data.update({'version': self.project.version(branch), 'branch': 'experimental'}) self.build_info.version(self.project.version(branch)) self.build_info.release('experimental:%s' % branch) open(os.path.join(self.builder.workdir, 'debian', 'changelog'), 'w').write(changelog_entry % changelog_data) else: self.build_info.version(force_version) self.build_info.release(force_release) rvm_env = {} rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: rvmexec = open(rvm_rc).read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: try: os.environ.pop('PATH') os.environ.pop('GEM_HOME') os.environ.pop('BUNDLER_PATH') except Exception, e: pass rvm_env.update(os.environ) os.chmod(os.path.join(debian_dir, 'rules'), stat.S_IRWXU|stat.S_IRWXG|stat.S_IROTH|stat.S_IXOTH) dpkg_cmd = self.builder._exec( ['dpkg-buildpackage', '-rfakeroot', '-tc', '-k%s' % BrickConfig().get('gpg', 'keyid')], cwd=self.builder.workdir, env=rvm_env, stdout=self.stdout, stderr=self.stderr ) dpkg_cmd.wait() clean_cmd = self.builder._exec(['dh', 'clean'], cwd=self.builder.workdir) clean_cmd.wait() if self.builder.build_options.changelog and os.path.isfile("%s.save" % changelog): os.rename("%s.save" % changelog, changelog)