def get(self, project_name): project = project_name build_ids = BuildInfo(project, -1).builds() builds = [] for bid in build_ids[-10:]: build = BuildInfo(project, bid) builds.append({'build': int(bid), 'log': os.path.basename(build.log()), 'version': build.version(), 'date': build.time()}) self.write(cyclone.escape.json_encode(builds))
def get(self, project_name): project = project_name build_ids = BuildInfo(project, -1).builds() builds = [] for bid in build_ids[-10:]: build = BuildInfo(project, bid) builds.append({ 'build': int(bid), 'log': os.path.basename(build.log()), 'version': build.version(), 'release': build.release(), 'date': build.time() }) self.write(cyclone.escape.json_encode(builds))
def get(self, project_name): project = project_name build_ids = BuildInfo(project, -1).builds() builds = [] for bid in build_ids[-10:]: build = BuildInfo(project, bid) builds.append( { "build": int(bid), "log": os.path.basename(build.log()), "version": build.version(), "release": build.release(), "date": build.time(), } ) self.write(cyclone.escape.json_encode(builds))
class BuilderDeb(): def __init__(self, builder): self.builder = builder self.project = self.builder.project log.info("Debian builder initialized: %s" % self.builder.workdir) log.info("Building [%s] with options: %s" % ( self.project.name, self.builder.build_options.options )) def configure_changelog(self, branch): template_data = { 'name': self.project.name, 'version': "%s" % (self.project.version(branch)), 'build_cmd': self.project.build_cmd, 'install_cmd': self.builder.mod_install_cmd, 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } changelog = os.path.join(self.builder.workdir, 'debian', 'changelog') if hasattr(self.builder.build_options, 'changelog') and self.builder.build_options.changelog: if os.path.isfile(changelog): os.rename(changelog, "%s.save" % changelog) def read_file_data(f): template_fd = open(os.path.join(templates_dir, f)) templates[f] = pystache.template.Template(template_fd.read()).render(context=template_data) template_fd.close() if not os.path.isdir(self.debian_dir): map(read_file_data, ['changelog', 'control', 'rules']) os.makedirs( os.path.join(self.debian_dir, self.project.name, self.project.install_prefix)) for filename, data in templates.iteritems(): open(os.path.join(self.debian_dir, filename), 'w').write(data) changelog_entry = """%(name)s (%(version)s) %(branch)s; urgency=low * Latest commits %(commits)s -- %(username)s <%(email)s> %(date)s """ changelog_data = { 'name': self.project.name, 'version': self.project.version(branch), 'branch': branch, 'commits': ' '.join(self.builder.git.log()), 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } return (changelog_entry, changelog_data) else: return (None, None) def build_install_deps(self): p = self.builder._exec(["dpkg-checkbuilddeps"], stderr=subprocess.PIPE) p.wait() out = p.stderr.read() if out != "": deps = re.findall("([a-z0-9\-]+\s|[a-z0-9\-]+$)", out.split("dependencies:")[1]) deps = map(lambda x: x.strip(), deps) apt_cmd = "apt-get -y --force-yes install %s" % " ".join(deps) b = self.builder._exec(apt_cmd.split(), stdout=self.stdout, stderr=self.stderr) b.wait() def build(self, branch, last_tag=None, force_version=None, force_release=None): templates = {} templates_dir = os.path.join(self.builder.templates_dir, 'deb') control_data_new = None self.build_info = BuildInfo(self.project.name) logfile = os.path.join( self.builder.workspace, 'log', '%s.%s.log' % ( self.project.name, self.build_info.build_id ) ) log.info("build log file: %s" % logfile) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout self.debian_dir = os.path.join(self.builder.workdir, 'debian') if last_tag is not None: os.environ.update({'BRICKLAYER_RELEASE': last_tag.split('_')[0]}) os.environ.update({'BRICKLAYER_TAG': last_tag}) # Not now #self.build_install_deps() if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd : self.project.install_cmd = 'cp -r \`ls | grep -v debian\` debian/tmp/%s' % ( self.project.install_prefix) changelog_entry, changelog_data = self.configure_changelog(branch) if changelog_entry and changelog_data: if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('stable'), 'branch': 'stable'}) self.build_info.version(self.project.version('stable')) self.build_info.release('stable') elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('testing'), 'branch': 'testing'}) self.build_info.version(self.project.version('testing')) self.build_info.release('testing') elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('unstable'), 'branch': 'unstable'}) self.build_info.version(self.project.version('unstable')) self.build_info.release('unstable') else: """ otherwise it should change the distribution to experimental """ version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) changelog_data.update({'version': self.project.version(branch), 'branch': 'experimental'}) self.build_info.version(self.project.version(branch)) self.build_info.release('experimental:%s' % branch) with file(os.path.join(self.builder.workdir, 'debian', 'changelog'), 'w') as fh: fh.write(changelog_entry % changelog_data) else: self.build_info.version(force_version) self.build_info.release(force_release) self.project.version(branch, force_version) self.project.version(force_release, force_version) rvm_env = {} rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: rvmexec = open(rvm_rc).read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/rvm/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: os_env = dict(os.environ) for k in ("PATH", "GEM_HOME", "BUNDLER_PATH"): if (k in os_env): del(os_env[k]) rvm_env.update(os_env) os.chmod(os.path.join(self.debian_dir, 'rules'), stat.S_IRWXU|stat.S_IRWXG|stat.S_IROTH|stat.S_IXOTH) dpkg_cmd = self.builder._exec( ['dpkg-buildpackage', '-rfakeroot', '-tc', '-k%s' % BrickConfig().get('gpg', 'keyid')], cwd=self.builder.workdir, env=rvm_env, stdout=self.stdout, stderr=self.stderr ) dpkg_cmd.wait() clean_cmd = self.builder._exec(['dh', 'clean'], cwd=self.builder.workdir, stdout=self.stdout, stderr=self.stderr) clean_cmd.wait() def upload(self, branch): glob_str = '%s/%s_%s_*.changes' % ( BrickConfig().get('workspace', 'dir'), self.project.name, self.project.version(branch)) changes_file = glob.glob(glob_str)[0] distribution, files = self.parse_changes(changes_file) self.local_repo(distribution, files) try: self.upload_files(distribution, files) upload_file = changes_file.replace('.changes', '.upload') open(upload_file, 'w').write("done") except Exception, e: log.error("Package could not be uploaded: %s", e)
def get(self, project, build): build_info = BuildInfo(project, build) if os.path.isfile(build_info.log()): self.write(open(build_info.log()).read())
def get(self, project, bid): build_info = BuildInfo(project, bid) if os.path.isfile(build_info.log()): self.write(open(build_info.log()).read())
class BuilderRpm(): def __init__(self, builder): self.builder = builder self.project = self.builder.project self.distribution = None self.version = None def dos2unix(self, file): f = open(file, 'r').readlines() new_file = open(file, "w+") match = re.compile('\r\n') for line in f: new_file.write(match.sub('\n', line)) new_file.close() def build(self, branch, last_tag=None): templates_dir = os.path.join(self.builder.templates_dir, 'rpm') rpm_dir = os.path.join(self.builder.workdir, 'redhat') spec_filename = os.path.join(rpm_dir, 'SPECS', "%s.spec" % self.project.name) self.build_info = BuildInfo(self.project.name) logfile = os.path.join(self.builder.workspace, 'log', '%s.%s.log' % (self.project.name, self.build_info.build_id)) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) self.build_info.version(self.project.version('stable')) self.version = self.project.version('stable') self.distribution = 'stable' elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) self.build_info.version(self.project.version('testing')) self.version = self.project.version('testing') self.distribution = 'testing' elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) self.build_info.version(self.project.version('unstable')) self.version = self.project.version('unstable') self.distribution = 'unstable' else: """ otherwise it should change the distribution to unstable """ if self.project.version(branch): version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) self.build_info.version(self.project.version(branch)) self.version = self.project.version(branch) self.distribution = 'experimental' dir_prefix = "%s-%s" % (self.project.name, self.version) for dir in ('SOURCES', 'SPECS', 'RPMS', 'SRPMS', 'BUILD', 'TMP'): if os.path.isdir(os.path.join(rpm_dir, dir)): shutil.rmtree(os.path.join(rpm_dir, dir)) os.makedirs(os.path.join(rpm_dir, dir)) build_dir = os.path.join(rpm_dir, 'TMP', self.project.name) os.makedirs(build_dir) if os.path.isdir(os.path.join(rpm_dir, dir_prefix)): shutil.rmtree(os.path.join(rpm_dir, dir_prefix)) os.makedirs(os.path.join(rpm_dir, dir_prefix)) subprocess.call(["cp -rP `ls -a | grep -Ev '\.$|\.\.$|debian$|redhat$'` %s" % os.path.join(rpm_dir, dir_prefix)], cwd=self.builder.workdir, shell=True ) cur_dir = os.getcwd() os.chdir(rpm_dir) source_file = os.path.join(rpm_dir, 'SOURCES', '%s.tar.gz' % dir_prefix) tar = tarfile.open(source_file, 'w:gz') tar.add(dir_prefix) tar.close() shutil.rmtree(dir_prefix) os.chdir(cur_dir) if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd: self.project.install_cmd = 'cp -r \`ls -a | grep -Ev "\.$|\.\.$|debian$"\` %s/%s/%s' % ( build_dir, self.project.install_prefix, self.project.name ) template_data = { 'name': self.project.name, 'version': self.version, 'build_dir': build_dir, 'build_cmd': self.project.build_cmd, 'install_cmd': self.builder.mod_install_cmd, 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a %h %d %Y"), 'git_url': self.project.git_url, 'source': source_file, } rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False environment = None if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: rvmexec = open(rvm_rc).read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/rvm/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() rvm_env = {} for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: try: os.environ.pop('PATH') os.environ.pop('GEM_HOME') os.environ.pop('BUNDLER_PATH') except Exception, e: pass for param in os.environ.keys(): if param.find('PROXY') != -1: rvm_env[param] = os.environ[param] rvm_env.update(os.environ) environment = rvm_env log.info(environment) if os.path.isfile(os.path.join(self.builder.workdir, 'rpm', "%s.spec" % self.project.name)): self.dos2unix(os.path.join(self.builder.workdir, 'rpm', "%s.spec" % self.project.name)) template_fd = open(os.path.join(self.builder.workdir, 'rpm', "%s.spec" % self.project.name)) else: template_fd = open(os.path.join(templates_dir, 'project.spec')) rendered_template = open(spec_filename, 'w+') rendered_template.write(pystache.template.Template(template_fd.read()).render(context=template_data)) template_fd.close() rendered_template.close() rendered_template = open(spec_filename, 'a') rendered_template.write("* %(date)s %(username)s <%(email)s> - %(version)s-1\n" % template_data) for git_log in self.builder.git.log(): rendered_template.write('- %s' % git_log) rendered_template.close() self.project.save() if type(environment) is NoneType: environment = os.environ rpm_cmd = self.builder._exec([ "rpmbuild", "--define", "_topdir %s" % rpm_dir, "-ba", spec_filename ], cwd=self.builder.workdir, env=environment, stdout=self.stdout, stderr=self.stderr ) rpm_cmd.wait() for path, dirs, files in os.walk(rpm_dir): if os.path.isdir(path): for file in (os.path.join(path, file) for file in files): try: if os.path.isfile(file) and file.endswith('.rpm'): shutil.copy(file, self.builder.workspace) except Exception, e: log.error(e)
class BuilderDeb(): def __init__(self, builder): self.builder = builder self.project = self.builder.project log.info("Debian builder initialized: %s" % self.builder.workdir) log.info("Building [%s] with options: %s" % ( self.project.name, self.builder.build_options.options )) def configure_changelog(self, branch): template_data = { 'name': self.project.name, 'version': "%s" % (self.project.version(branch)), 'build_cmd': self.project.build_cmd, 'install_cmd': self.builder.mod_install_cmd, 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } changelog = os.path.join(self.builder.workdir, 'debian', 'changelog') if hasattr(self.builder.build_options, 'changelog') and self.builder.build_options.changelog: if os.path.isfile(changelog): os.rename(changelog, "%s.save" % changelog) def read_file_data(f): with open(os.path.join(templates_dir, f)) as tmpfh: templates[f] = pystache.template.Template(tmpfh.read()).render(context=template_data) if not os.path.isdir(self.debian_dir): map(read_file_data, ['changelog', 'control', 'rules']) os.makedirs( os.path.join(self.debian_dir, self.project.name, self.project.install_prefix)) for filename, data in templates.iteritems(): with open(os.path.join(self.debian_dir, filename), 'w') as tmpfh: tmpfh.write(data) changelog_entry = """%(name)s (%(version)s) %(branch)s; urgency=low * Latest commits %(commits)s -- %(username)s <%(email)s> %(date)s """ changelog_data = { 'name': self.project.name, 'version': self.project.version(branch), 'branch': branch, 'commits': ' '.join(self.builder.git.log()), 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } return (changelog_entry, changelog_data) else: return (None, None) def build_install_deps(self): p = self.builder._exec(["dpkg-checkbuilddeps"], stderr=subprocess.PIPE, close_fds=True) p.wait() out = p.stderr.read() if out != "": deps = re.findall("([a-z0-9\-]+\s|[a-z0-9\-]+$)", out.split("dependencies:")[1]) deps = map(lambda x: x.strip(), deps) apt_cmd = "apt-get -y --force-yes install %s" % " ".join(deps) b = self.builder._exec(apt_cmd.split(), stdout=self.stdout, stderr=self.stderr, close_fds=True) b.wait() def build(self, branch, last_tag=None, force_version=None, force_release=None): templates = {} templates_dir = os.path.join(self.builder.templates_dir, 'deb') control_data_new = None self.build_info = BuildInfo(self.project.name) logfile = os.path.join( BrickConfig().get("workspace", "dir"), 'log', '%s.%s.log' % ( self.project.name, self.build_info.build_id ) ) log.info("build log file: %s" % logfile) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout self.debian_dir = os.path.join(self.builder.workdir, 'debian') if last_tag is not None: os.environ.update({'BRICKLAYER_RELEASE': last_tag.split('_')[0]}) os.environ.update({'BRICKLAYER_TAG': last_tag}) # Not now #self.build_install_deps() if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd : self.project.install_cmd = 'cp -r \`ls | grep -v debian\` debian/tmp/%s' % ( self.project.install_prefix ) changelog_entry, changelog_data = self.configure_changelog(branch) if changelog_entry and changelog_data: if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('stable'), 'branch': 'stable'}) self.build_info.version(self.project.version('stable')) self.build_info.release('stable') elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('testing'), 'branch': 'testing'}) self.build_info.version(self.project.version('testing')) self.build_info.release('testing') elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('unstable'), 'branch': 'unstable'}) self.build_info.version(self.project.version('unstable')) self.build_info.release('unstable') else: """ otherwise it should change the distribution to experimental """ version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) changelog_data.update({'version': self.project.version(branch), 'branch': 'experimental'}) self.build_info.version(self.project.version(branch)) self.build_info.release('experimental:%s' % branch) with file(os.path.join(self.builder.workdir, 'debian', 'changelog'), 'w') as fh: fh.write(changelog_entry % changelog_data) else: self.build_info.version(force_version) self.build_info.release(force_release) self.project.version(branch, force_version) self.project.version(force_release, force_version) rvm_env = {} rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: with open(rvm_rc) as tmpfh: rvmexec = tmpfh.read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/rvm/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: os_env = dict(os.environ) for k in ("PATH", "GEM_HOME", "BUNDLER_PATH"): if (k in os_env): del(os_env[k]) rvm_env.update(os_env) log.info(rvm_env) os.chmod(os.path.join(self.debian_dir, 'rules'), stat.S_IRWXU|stat.S_IRWXG|stat.S_IROTH|stat.S_IXOTH) dpkg_cmd = self.builder._exec( ['dpkg-buildpackage', '-rfakeroot', '-tc', '-k%s' % BrickConfig().get('gpg', 'keyid')], cwd=self.builder.workdir, env=rvm_env, stdout=self.stdout, stderr=self.stderr, close_fds=True) dpkg_cmd.wait() clean_cmd = self.builder._exec(['dh', 'clean'], cwd=self.builder.workdir, stdout=self.stdout, stderr=self.stderr, close_fds=True) clean_cmd.wait() def upload(self, branch): glob_str = '%s/%s_%s_*.changes' % ( BrickConfig().get('workspace', 'dir'), self.project.name, self.project.version(branch)) changes_file = glob.glob(glob_str) log.info(changes_file) distribution, files = self.parse_changes(changes_file[0]) try: self.upload_files(distribution, files) upload_file = changes_file[0].replace('.changes', '.upload') with open(upload_file, 'w') as tmpfh: tmpfh.write("done") except Exception, e: log.error("Package could not be uploaded: %s", e)
class BuilderDeb: def __init__(self, builder): self.builder = builder self.project = self.builder.project log.msg("Building [%s] with options: %s" % (self.project.name, self.builder.build_options.options)) def configure_changelog(self, branch): template_data = { "name": self.project.name, "version": "%s" % (self.project.version(branch)), "build_cmd": self.project.build_cmd, "install_cmd": self.builder.mod_install_cmd, "username": self.project.username, "email": self.project.email, "date": time.strftime("%a, %d %h %Y %T %z"), } changelog = os.path.join(self.builder.workdir, "debian", "changelog") if hasattr(self.builder.build_options, "changelog") and self.builder.build_options.changelog: if os.path.isfile(changelog): os.rename(changelog, "%s.save" % changelog) def read_file_data(f): with open(os.path.join(templates_dir, f)) as tmpfh: templates[f] = pystache.template.Template(tmpfh.read()).render(context=template_data) if not os.path.isdir(self.debian_dir): map(read_file_data, ["changelog", "control", "rules"]) os.makedirs(os.path.join(self.debian_dir, self.project.name, self.project.install_prefix)) for filename, data in templates.iteritems(): with open(os.path.join(self.debian_dir, filename), "w") as tmpfh: tmpfh.write(data) changelog_entry = """%(name)s (%(version)s) %(branch)s; urgency=low * Latest commits %(commits)s -- %(username)s <%(email)s> %(date)s """ changelog_data = { "name": self.project.name, "version": self.project.version(branch), "branch": branch, "commits": " ".join(self.builder.git.log()), "username": self.project.username, "email": self.project.email, "date": time.strftime("%a, %d %h %Y %T %z"), } return (changelog_entry, changelog_data) else: return (None, None) def build_install_deps(self): p = self.builder._exec(["dpkg-checkbuilddeps"], stderr=subprocess.PIPE, close_fds=True) p.wait() out = p.stderr.read() if out != "": deps = re.findall("([a-z0-9\-]+\s|[a-z0-9\-]+$)", out.split("dependencies:")[1]) deps = map(lambda x: x.strip(), deps) apt_cmd = "apt-get -y --force-yes install %s" % " ".join(deps) b = self.builder._exec(apt_cmd.split(), stdout=self.stdout, stderr=self.stderr, close_fds=True) b.wait() def build(self, branch, last_tag=None, force_version=None, force_release=None): templates = {} templates_dir = os.path.join(self.builder.templates_dir, "deb") control_data_new = None self.build_info = BuildInfo(self.project.name) logfile = os.path.join( BrickConfig().get("workspace", "dir"), "log", "%s.%s.log" % (self.project.name, self.build_info.build_id) ) self.build_info.log(logfile) self.stdout = open(logfile, "a+") self.stderr = self.stdout self.debian_dir = os.path.join(self.builder.workdir, "debian") if last_tag is not None: os.environ.update({"BRICKLAYER_RELEASE": last_tag.split("_")[0]}) os.environ.update({"BRICKLAYER_TAG": last_tag}) # Not now # self.build_install_deps() if self.project.install_prefix is None: self.project.install_prefix = "opt" if not self.project.install_cmd: self.project.install_cmd = "cp -r \`ls | grep -v debian\` debian/tmp/%s" % (self.project.install_prefix) changelog_entry, changelog_data = self.configure_changelog(branch) if changelog_entry and changelog_data: if last_tag != None and last_tag.startswith("stable"): self.project.version("stable", last_tag.split("_")[1]) changelog_data.update({"version": self.project.version("stable"), "branch": "stable"}) self.build_info.version(self.project.version("stable")) self.build_info.release("stable") elif last_tag != None and last_tag.startswith("testing"): self.project.version("testing", last_tag.split("_")[1]) changelog_data.update({"version": self.project.version("testing"), "branch": "testing"}) self.build_info.version(self.project.version("testing")) self.build_info.release("testing") elif last_tag != None and last_tag.startswith("unstable"): self.project.version("unstable", last_tag.split("_")[1]) changelog_data.update({"version": self.project.version("unstable"), "branch": "unstable"}) self.build_info.version(self.project.version("unstable")) self.build_info.release("unstable") else: """ otherwise it should change the distribution to experimental """ version_list = self.project.version(branch).split(".") version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, ".".join(version_list)) changelog_data.update({"version": self.project.version(branch), "branch": "experimental"}) self.build_info.version(self.project.version(branch)) self.build_info.release("experimental:%s" % branch) with file(os.path.join(self.builder.workdir, "debian", "changelog"), "w") as fh: fh.write(changelog_entry % changelog_data) else: self.build_info.version(force_version) self.build_info.release(force_release) self.project.version(branch, force_version) self.project.version(force_release, force_version) rvm_rc = os.path.join(self.builder.workdir, ".rvmrc") rvm_rc_example = rvm_rc + ".example" has_rvm = False has_rvm = os.path.isfile(rvm_rc) or os.path.isfile(rvm_rc_example) os.chmod(os.path.join(self.debian_dir, "rules"), stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH) if has_rvm: dpkg_cmd = self.builder._exec( [ "/usr/bin/bricklayer-rvm-exec", "/usr/bin/dpkg-buildpackage", "-rfakeroot", "-tc", "-k%s" % BrickConfig().get("gpg", "keyid"), ], cwd=self.builder.workdir, stdout=self.stdout, stderr=self.stderr, close_fds=True, ) else: dpkg_cmd = self.builder._exec( ["dpkg-buildpackage", "-rfakeroot", "-tc", "-k%s" % BrickConfig().get("gpg", "keyid")], cwd=self.builder.workdir, stdout=self.stdout, stderr=self.stderr, close_fds=True, ) dpkg_cmd.wait() clean_cmd = self.builder._exec( ["dh", "clean"], cwd=self.builder.workdir, stdout=self.stdout, stderr=self.stderr, close_fds=True ) clean_cmd.wait() def upload(self, branch): glob_str = "%s/%s_%s_*.changes" % ( BrickConfig().get("workspace", "dir"), self.project.name, self.project.version(branch), ) changes_file = glob.glob(glob_str) distribution, files = self.parse_changes(changes_file[0]) try: self.upload_files(distribution, files) upload_file = changes_file[0].replace(".changes", ".upload") with open(upload_file, "w") as tmpfh: tmpfh.write("done") except Exception, e: log.msg("[%s] %s" % (self.project.name, traceback.format_exc()))
class DebBuilder(): def __init__(self, builder): self.builder = builder self.project = self.builder.project log.info("Debian builder initialized: %s" % self.builder.workdir) def build(self, branch, last_tag=None): templates = {} templates_dir = os.path.join(self.builder.templates_dir, 'deb') debian_dir = os.path.join(self.builder.workdir, 'debian') control_data_new = None self.build_info = BuildInfo(self.project.name) logfile = os.path.join(self.builder.workspace, 'log', '%s.%s.log' % (self.project.name, self.build_info.build_id)) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd : self.project.install_cmd = 'cp -r \`ls | grep -v debian\` debian/tmp/%s' % ( self.project.install_prefix ) template_data = { 'name': self.project.name, 'version': "%s" % (self.project.version(branch)), 'build_cmd': self.project.build_cmd, 'install_cmd': self.builder.mod_install_cmd, 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } changelog = os.path.join(self.builder.workdir, 'debian', 'changelog') if os.path.isfile(changelog): os.rename(changelog, "%s.save" % changelog) def read_file_data(f): template_fd = open(os.path.join(templates_dir, f)) templates[f] = pystache.template.Template(template_fd.read()).render(context=template_data) template_fd.close() if not os.path.isdir(debian_dir): map(read_file_data, ['changelog', 'control', 'rules']) os.makedirs( os.path.join( debian_dir, self.project.name, self.project.install_prefix)) for filename, data in templates.iteritems(): open(os.path.join(debian_dir, filename), 'w').write(data) changelog_entry = """%(name)s (%(version)s) %(branch)s; urgency=low * Latest commits %(commits)s -- %(username)s <%(email)s> %(date)s """ changelog_data = { 'name': self.project.name, 'version': self.project.version(branch), 'branch': branch, 'commits': ' '.join(self.builder.git.log()), 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('stable'), 'branch': 'stable'}) self.build_info.version(self.project.version('stable')) self.build_info.release('stable') elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('testing'), 'branch': 'testing'}) self.build_info.version(self.project.version('testing')) self.build_info.release('testing') elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('unstable'), 'branch': 'unstable'}) self.build_info.version(self.project.version('unstable')) self.build_info.release('unstable') else: """ otherwise it should change the distribution to experimental """ if self.project.version(branch): version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) changelog_data.update({'version': self.project.version(branch), 'branch': 'experimental'}) self.build_info.version(self.project.version(branch)) self.build_info.release('experimental:%s' % branch) open(os.path.join(self.builder.workdir, 'debian', 'changelog'), 'w').write(changelog_entry % changelog_data) rvm_env = {} rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: rvmexec = open(rvm_rc).read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: try: os.environ.pop('PATH') os.environ.pop('GEM_HOME') os.environ.pop('BUNDLER_PATH') except Exception, e: pass rvm_env.update(os.environ) os.chmod(os.path.join(debian_dir, 'rules'), stat.S_IRWXU|stat.S_IRWXG|stat.S_IROTH|stat.S_IXOTH) dpkg_cmd = self.builder._exec( ['dpkg-buildpackage', '-rfakeroot', '-tc', '-k%s' % BrickConfig().get('gpg', 'keyid')], cwd=self.builder.workdir, env=rvm_env, stdout=self.stdout, stderr=self.stderr ) dpkg_cmd.wait() clean_cmd = self.builder._exec(['dh', 'clean'], cwd=self.builder.workdir) clean_cmd.wait() if os.path.isfile("%s.save" % changelog): os.rename("%s.save" % changelog, changelog)
class BuilderRpm(): def __init__(self, builder): self.builder = builder self.project = self.builder.project self.distribution = None self.version = None def dos2unix(self, file): f = open(file, 'r').readlines() new_file = open(file, "w+") match = re.compile('\r\n') for line in f: new_file.write(match.sub('\n', line)) new_file.close() def build(self, branch, last_tag=None): templates_dir = os.path.join(self.builder.templates_dir, 'rpm') rpm_dir = os.path.join(self.builder.workdir, 'redhat') spec_filename = os.path.join(rpm_dir, 'SPECS', "%s.spec" % self.project.name) self.build_info = BuildInfo(self.project.name) logfile = os.path.join( self.builder.workspace, 'log', '%s.%s.log' % (self.project.name, self.build_info.build_id)) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) self.build_info.version(self.project.version('stable')) self.version = self.project.version('stable') self.distribution = 'stable' elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) self.build_info.version(self.project.version('testing')) self.version = self.project.version('testing') self.distribution = 'testing' elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) self.build_info.version(self.project.version('unstable')) self.version = self.project.version('unstable') self.distribution = 'unstable' else: """ otherwise it should change the distribution to unstable """ if self.project.version(branch): version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str( int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) self.build_info.version(self.project.version(branch)) self.version = self.project.version(branch) self.distribution = 'experimental' dir_prefix = "%s-%s" % (self.project.name, self.version) for dir in ('SOURCES', 'SPECS', 'RPMS', 'SRPMS', 'BUILD', 'TMP'): if os.path.isdir(os.path.join(rpm_dir, dir)): shutil.rmtree(os.path.join(rpm_dir, dir)) os.makedirs(os.path.join(rpm_dir, dir)) build_dir = os.path.join(rpm_dir, 'TMP', self.project.name) os.makedirs(build_dir) if os.path.isdir(os.path.join(rpm_dir, dir_prefix)): shutil.rmtree(os.path.join(rpm_dir, dir_prefix)) os.makedirs(os.path.join(rpm_dir, dir_prefix)) subprocess.call([ "cp -rP `ls -a | grep -Ev '\.$|\.\.$|debian$|redhat$'` %s" % os.path.join(rpm_dir, dir_prefix) ], cwd=self.builder.workdir, shell=True) cur_dir = os.getcwd() os.chdir(rpm_dir) source_file = os.path.join(rpm_dir, 'SOURCES', '%s.tar.gz' % dir_prefix) tar = tarfile.open(source_file, 'w:gz') tar.add(dir_prefix) tar.close() shutil.rmtree(dir_prefix) os.chdir(cur_dir) if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd: self.project.install_cmd = 'cp -r \`ls -a | grep -Ev "\.$|\.\.$|debian$"\` %s/%s/%s' % ( build_dir, self.project.install_prefix, self.project.name) template_data = { 'name': self.project.name, 'version': self.version, 'build_dir': build_dir, 'build_cmd': self.project.build_cmd, 'install_cmd': self.builder.mod_install_cmd, 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a %h %d %Y"), 'git_url': self.project.git_url, 'source': source_file, } rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False environment = None if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: rvmexec = open(rvm_rc).read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/rvm/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() rvm_env = {} for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: try: os.environ.pop('PATH') os.environ.pop('GEM_HOME') os.environ.pop('BUNDLER_PATH') except Exception, e: pass for param in os.environ.keys(): if param.find('PROXY') != -1: rvm_env[param] = os.environ[param] rvm_env.update(os.environ) environment = rvm_env log.info(environment) if os.path.isfile( os.path.join(self.builder.workdir, 'rpm', "%s.spec" % self.project.name)): self.dos2unix( os.path.join(self.builder.workdir, 'rpm', "%s.spec" % self.project.name)) template_fd = open( os.path.join(self.builder.workdir, 'rpm', "%s.spec" % self.project.name)) else: template_fd = open(os.path.join(templates_dir, 'project.spec')) rendered_template = open(spec_filename, 'w+') rendered_template.write( pystache.template.Template( template_fd.read()).render(context=template_data)) template_fd.close() rendered_template.close() rendered_template = open(spec_filename, 'a') rendered_template.write( "* %(date)s %(username)s <%(email)s> - %(version)s-1\n" % template_data) for git_log in self.builder.git.log(): rendered_template.write('- %s' % git_log) rendered_template.close() self.project.save() if type(environment) is NoneType: environment = os.environ rpm_cmd = self.builder._exec([ "rpmbuild", "--define", "_topdir %s" % rpm_dir, "-ba", spec_filename ], cwd=self.builder.workdir, env=environment, stdout=self.stdout, stderr=self.stderr) rpm_cmd.wait() for path, dirs, files in os.walk(rpm_dir): if os.path.isdir(path): for file in (os.path.join(path, file) for file in files): try: if os.path.isfile(file) and file.endswith('.rpm'): shutil.copy(file, self.builder.workspace) except Exception, e: log.error(e)
class BuilderDeb(): def __init__(self, builder): self.builder = builder self.project = self.builder.project log.info("Debian builder initialized: %s" % self.builder.workdir) log.info("Building with options: %s" % self.builder.build_options.options) def build(self, branch, last_tag=None, force_version=None, force_release=None): templates = {} templates_dir = os.path.join(self.builder.templates_dir, 'deb') debian_dir = os.path.join(self.builder.workdir, 'debian') control_data_new = None self.build_info = BuildInfo(self.project.name) logfile = os.path.join( self.builder.workspace, 'log', '%s.%s.log' % ( self.project.name, self.build_info.build_id ) ) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd : self.project.install_cmd = 'cp -r \`ls | grep -v debian\` debian/tmp/%s' % ( self.project.install_prefix ) template_data = { 'name': self.project.name, 'version': "%s" % (self.project.version(branch)), 'build_cmd': self.project.build_cmd, 'install_cmd': self.builder.mod_install_cmd, 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } changelog = os.path.join(self.builder.workdir, 'debian', 'changelog') if hasattr(self.builder.build_options, 'changelog') and self.builder.build_options.changelog: if os.path.isfile(changelog): os.rename(changelog, "%s.save" % changelog) def read_file_data(f): template_fd = open(os.path.join(templates_dir, f)) templates[f] = pystache.template.Template(template_fd.read()).render(context=template_data) template_fd.close() if not os.path.isdir(debian_dir): map(read_file_data, ['changelog', 'control', 'rules']) os.makedirs( os.path.join( debian_dir, self.project.name, self.project.install_prefix)) for filename, data in templates.iteritems(): open(os.path.join(debian_dir, filename), 'w').write(data) changelog_entry = """%(name)s (%(version)s) %(branch)s; urgency=low * Latest commits %(commits)s -- %(username)s <%(email)s> %(date)s """ changelog_data = { 'name': self.project.name, 'version': self.project.version(branch), 'branch': branch, 'commits': ' '.join(self.builder.git.log()), 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('stable'), 'branch': 'stable'}) self.build_info.version(self.project.version('stable')) self.build_info.release('stable') elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('testing'), 'branch': 'testing'}) self.build_info.version(self.project.version('testing')) self.build_info.release('testing') elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('unstable'), 'branch': 'unstable'}) self.build_info.version(self.project.version('unstable')) self.build_info.release('unstable') else: """ otherwise it should change the distribution to experimental """ if self.project.version(branch): version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) changelog_data.update({'version': self.project.version(branch), 'branch': 'experimental'}) self.build_info.version(self.project.version(branch)) self.build_info.release('experimental:%s' % branch) open(os.path.join(self.builder.workdir, 'debian', 'changelog'), 'w').write(changelog_entry % changelog_data) else: self.build_info.version(force_version) self.build_info.release(force_release) rvm_env = {} rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: rvmexec = open(rvm_rc).read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: try: os.environ.pop('PATH') os.environ.pop('GEM_HOME') os.environ.pop('BUNDLER_PATH') except Exception, e: pass rvm_env.update(os.environ) os.chmod(os.path.join(debian_dir, 'rules'), stat.S_IRWXU|stat.S_IRWXG|stat.S_IROTH|stat.S_IXOTH) dpkg_cmd = self.builder._exec( ['dpkg-buildpackage', '-rfakeroot', '-tc', '-k%s' % BrickConfig().get('gpg', 'keyid')], cwd=self.builder.workdir, env=rvm_env, stdout=self.stdout, stderr=self.stderr ) dpkg_cmd.wait() clean_cmd = self.builder._exec(['dh', 'clean'], cwd=self.builder.workdir) clean_cmd.wait() if self.builder.build_options.changelog and os.path.isfile("%s.save" % changelog): os.rename("%s.save" % changelog, changelog)