def get(self, project_name): project = project_name build_ids = BuildInfo(project, -1).builds() builds = [] for bid in build_ids[-10:]: build = BuildInfo(project, bid) builds.append({'build': int(bid), 'log': os.path.basename(build.log()), 'version': build.version(), 'date': build.time()}) self.write(cyclone.escape.json_encode(builds))
def process_code_build(event): if 'additional-information' not in event['detail']: logger.debug("Skipping due to no additional-information") return cbi = CodeBuildInfo.from_event(event) logger.debug(vars(cbi)) (stage, pid, actionStates) = pipeline_from_build(cbi) logger.debug(stage, pid, actionStates) if not pid: return build_info = BuildInfo(pid, cbi.pipeline) existing_msg = find_message_for_build(build_info) builder = MessageBuilder(build_info, existing_msg) if 'phases' in event['detail']['additional-information']: phases = event['detail']['additional-information']['phases'] builder.update_build_stage_info(stage, phases, actionStates) logs = event['detail'].get('additional-information', {}).get('logs') post_build_msg(builder)
def processCodeBuild(event): # logger.info("processCodeBuild") if is_skip_codebuild_notice(event['detail'].get('eventName')): return cbi = CodeBuildInfo.from_event(event) (stage, pid, actionStates) = pipelineFromBuild(cbi) if not pid: return build_info = BuildInfo(pid, cbi.pipeline, actionStates) existing_msg = find_message_for_build(build_info) builder = MessageBuilder(build_info, existing_msg) if 'phases' in event['detail']['additional-information']: phases = event['detail']['additional-information']['phases'] builder.updateBuildStageInfo(stage, phases, actionStates) logs = event['detail'].get('additional-information', {}).get('logs') if logs: builder.attachLogs(event['detail']['additional-information']['logs']) post_build_msg(builder)
def get(self, project_name): project = project_name build_ids = BuildInfo(project, -1).builds() builds = [] for bid in build_ids[-10:]: build = BuildInfo(project, bid) builds.append( { "build": int(bid), "log": os.path.basename(build.log()), "version": build.version(), "release": build.release(), "date": build.time(), } ) self.write(cyclone.escape.json_encode(builds))
def processCodeBuild(event): event_id = event['detail']['build-id'].split('/')[1] build_status = codebuild_client.batch_get_builds(ids=[event_id]) pid = event_id.split(':')[1] or None if not pid: return pipeline_name = event_id.split(':')[0] cbi = CodeBuildInfo(build_status['builds'][0]['initiator'][13:], event_id) (stage, pid, actionStates) = pipelineFromBuild(cbi) buildInfo = BuildInfo(pid, pipeline_name) existing_msg = find_message_for_build(buildInfo) builder = MessageBuilder(buildInfo, existing_msg) phases = build_status['builds'][0]['phases'] builder.updateBuildStageInfo(stage, phases, actionStates) logs = build_status['builds'][0].get('logs', {}) try: if logs['streamName']: builder.attachLogs(logs) except KeyError: pass post_build_msg(builder)
def parse(event): if event['source'] == 'aws.codebuild': ini = event['detail']['additional-information']['initiator'] m = re.match('codepipeline/(.+)', ini) if m: pipeline = m.group(1) r = client.get_pipeline_state(name=pipeline) buildInfo = BuildInfo.fromEvent(event) return buildInfo
def processCodePipeline(event): buildInfo = BuildInfo.fromEvent(event) existing_msg = find_message_for_build(buildInfo) builder = MessageBuilder(buildInfo, existing_msg) builder.updatePipelineEvent(event) if builder.needsRevisionInfo(): revision = findRevisionInfo(buildInfo) builder.attachRevisionInfo(revision) post_build_msg(builder)
def process_code_pipeline(event): if 'execution-id' not in event['detail']: logger.debug("Skipping due to no executionId") return build_info = BuildInfo.from_event(event) existing_msg = find_message_for_build(build_info) builder = MessageBuilder(build_info, existing_msg) builder.update_pipeline_event(event) if builder.needs_revision_info(): revision = find_revision_info(build_info) builder.attach_revision_info(revision) post_build_msg(builder)
def get(self, project_name): project = project_name build_ids = BuildInfo(project, -1).builds() builds = [] for bid in build_ids[-10:]: build = BuildInfo(project, bid) builds.append({ 'build': int(bid), 'log': os.path.basename(build.log()), 'version': build.version(), 'release': build.release(), 'date': build.time() }) self.write(cyclone.escape.json_encode(builds))
def processCodePipeline(event): # logger.info("processCodePipeline") if is_skip_codepipeline_notice(event['detail'].get('eventName')): return build_info = BuildInfo.from_event(event) existing_msg = find_message_for_build(build_info) # logger.info("existing_msg: {}".format(existing_msg)) builder = MessageBuilder(build_info, existing_msg) builder.updatePipelineEvent(event) if builder.needsRevisionInfo(): revision = findRevisionInfo(build_info) builder.attachRevisionInfo(revision) post_build_msg(builder)
def processCodeBuild(event): cbi = CodeBuildInfo.fromEvent(event) (stage, pid, actionStates) = pipelineFromBuild(cbi) if not pid: return buildInfo = BuildInfo(pid, cbi.pipeline) existing_msg = find_message_for_build(buildInfo) builder = MessageBuilder(buildInfo, existing_msg) if 'phases' in event['detail']['additional-information']: phases = event['detail']['additional-information']['phases'] builder.updateBuildStageInfo(stage, phases, actionStates) # logs = event['detail'].get('additional-information', {}).get('logs') # if logs: # builder.attachLogs(event['detail']['additional-information']['logs']) post_build_msg(builder)
def build(self, branch, last_tag=None): templates_dir = os.path.join(self.builder.templates_dir, 'rpm') rpm_dir = os.path.join(self.builder.workdir, 'redhat') spec_filename = os.path.join(rpm_dir, 'SPECS', "%s.spec" % self.project.name) self.build_info = BuildInfo(self.project.name) logfile = os.path.join( self.builder.workspace, 'log', '%s.%s.log' % (self.project.name, self.build_info.build_id)) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) self.build_info.version(self.project.version('stable')) self.version = self.project.version('stable') self.distribution = 'stable' elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) self.build_info.version(self.project.version('testing')) self.version = self.project.version('testing') self.distribution = 'testing' elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) self.build_info.version(self.project.version('unstable')) self.version = self.project.version('unstable') self.distribution = 'unstable' else: """ otherwise it should change the distribution to unstable """ if self.project.version(branch): version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str( int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) self.build_info.version(self.project.version(branch)) self.version = self.project.version(branch) self.distribution = 'experimental' dir_prefix = "%s-%s" % (self.project.name, self.version) for dir in ('SOURCES', 'SPECS', 'RPMS', 'SRPMS', 'BUILD', 'TMP'): if os.path.isdir(os.path.join(rpm_dir, dir)): shutil.rmtree(os.path.join(rpm_dir, dir)) os.makedirs(os.path.join(rpm_dir, dir)) build_dir = os.path.join(rpm_dir, 'TMP', self.project.name) os.makedirs(build_dir) if os.path.isdir(os.path.join(rpm_dir, dir_prefix)): shutil.rmtree(os.path.join(rpm_dir, dir_prefix)) os.makedirs(os.path.join(rpm_dir, dir_prefix)) subprocess.call([ "cp -rP `ls -a | grep -Ev '\.$|\.\.$|debian$|redhat$'` %s" % os.path.join(rpm_dir, dir_prefix) ], cwd=self.builder.workdir, shell=True) cur_dir = os.getcwd() os.chdir(rpm_dir) source_file = os.path.join(rpm_dir, 'SOURCES', '%s.tar.gz' % dir_prefix) tar = tarfile.open(source_file, 'w:gz') tar.add(dir_prefix) tar.close() shutil.rmtree(dir_prefix) os.chdir(cur_dir) if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd: self.project.install_cmd = 'cp -r \`ls -a | grep -Ev "\.$|\.\.$|debian$"\` %s/%s/%s' % ( build_dir, self.project.install_prefix, self.project.name) template_data = { 'name': self.project.name, 'version': self.version, 'build_dir': build_dir, 'build_cmd': self.project.build_cmd, 'install_cmd': self.builder.mod_install_cmd, 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a %h %d %Y"), 'git_url': self.project.git_url, 'source': source_file, } rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False environment = None if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: rvmexec = open(rvm_rc).read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/rvm/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() rvm_env = {} for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: try: os.environ.pop('PATH') os.environ.pop('GEM_HOME') os.environ.pop('BUNDLER_PATH') except Exception, e: pass for param in os.environ.keys(): if param.find('PROXY') != -1: rvm_env[param] = os.environ[param] rvm_env.update(os.environ) environment = rvm_env log.info(environment)
def build(self, branch, last_tag=None): templates_dir = os.path.join(self.builder.templates_dir, 'rpm') rpm_dir = os.path.join(self.builder.workdir, 'redhat') spec_filename = os.path.join(rpm_dir, 'SPECS', "%s.spec" % self.project.name) self.build_info = BuildInfo(self.project.name) logfile = os.path.join(self.builder.workspace, 'log', '%s.%s.log' % (self.project.name, self.build_info.build_id)) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) self.build_info.version(self.project.version('stable')) self.version = self.project.version('stable') self.distribution = 'stable' elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) self.build_info.version(self.project.version('testing')) self.version = self.project.version('testing') self.distribution = 'testing' elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) self.build_info.version(self.project.version('unstable')) self.version = self.project.version('unstable') self.distribution = 'unstable' else: """ otherwise it should change the distribution to unstable """ if self.project.version(branch): version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) self.build_info.version(self.project.version(branch)) self.version = self.project.version(branch) self.distribution = 'experimental' dir_prefix = "%s-%s" % (self.project.name, self.version) for dir in ('SOURCES', 'SPECS', 'RPMS', 'SRPMS', 'BUILD', 'TMP'): if os.path.isdir(os.path.join(rpm_dir, dir)): shutil.rmtree(os.path.join(rpm_dir, dir)) os.makedirs(os.path.join(rpm_dir, dir)) build_dir = os.path.join(rpm_dir, 'TMP', self.project.name) os.makedirs(build_dir) if os.path.isdir(os.path.join(rpm_dir, dir_prefix)): shutil.rmtree(os.path.join(rpm_dir, dir_prefix)) os.makedirs(os.path.join(rpm_dir, dir_prefix)) subprocess.call(["cp -rP `ls -a | grep -Ev '\.$|\.\.$|debian$|redhat$'` %s" % os.path.join(rpm_dir, dir_prefix)], cwd=self.builder.workdir, shell=True ) cur_dir = os.getcwd() os.chdir(rpm_dir) source_file = os.path.join(rpm_dir, 'SOURCES', '%s.tar.gz' % dir_prefix) tar = tarfile.open(source_file, 'w:gz') tar.add(dir_prefix) tar.close() shutil.rmtree(dir_prefix) os.chdir(cur_dir) if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd: self.project.install_cmd = 'cp -r \`ls -a | grep -Ev "\.$|\.\.$|debian$"\` %s/%s/%s' % ( build_dir, self.project.install_prefix, self.project.name ) template_data = { 'name': self.project.name, 'version': self.version, 'build_dir': build_dir, 'build_cmd': self.project.build_cmd, 'install_cmd': self.builder.mod_install_cmd, 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a %h %d %Y"), 'git_url': self.project.git_url, 'source': source_file, } rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False environment = None if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: rvmexec = open(rvm_rc).read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/rvm/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() rvm_env = {} for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: try: os.environ.pop('PATH') os.environ.pop('GEM_HOME') os.environ.pop('BUNDLER_PATH') except Exception, e: pass for param in os.environ.keys(): if param.find('PROXY') != -1: rvm_env[param] = os.environ[param] rvm_env.update(os.environ) environment = rvm_env log.info(environment)
class BuilderRpm(): def __init__(self, builder): self.builder = builder self.project = self.builder.project self.distribution = None self.version = None def dos2unix(self, file): f = open(file, 'r').readlines() new_file = open(file, "w+") match = re.compile('\r\n') for line in f: new_file.write(match.sub('\n', line)) new_file.close() def build(self, branch, last_tag=None): templates_dir = os.path.join(self.builder.templates_dir, 'rpm') rpm_dir = os.path.join(self.builder.workdir, 'redhat') spec_filename = os.path.join(rpm_dir, 'SPECS', "%s.spec" % self.project.name) self.build_info = BuildInfo(self.project.name) logfile = os.path.join(self.builder.workspace, 'log', '%s.%s.log' % (self.project.name, self.build_info.build_id)) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) self.build_info.version(self.project.version('stable')) self.version = self.project.version('stable') self.distribution = 'stable' elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) self.build_info.version(self.project.version('testing')) self.version = self.project.version('testing') self.distribution = 'testing' elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) self.build_info.version(self.project.version('unstable')) self.version = self.project.version('unstable') self.distribution = 'unstable' else: """ otherwise it should change the distribution to unstable """ if self.project.version(branch): version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) self.build_info.version(self.project.version(branch)) self.version = self.project.version(branch) self.distribution = 'experimental' dir_prefix = "%s-%s" % (self.project.name, self.version) for dir in ('SOURCES', 'SPECS', 'RPMS', 'SRPMS', 'BUILD', 'TMP'): if os.path.isdir(os.path.join(rpm_dir, dir)): shutil.rmtree(os.path.join(rpm_dir, dir)) os.makedirs(os.path.join(rpm_dir, dir)) build_dir = os.path.join(rpm_dir, 'TMP', self.project.name) os.makedirs(build_dir) if os.path.isdir(os.path.join(rpm_dir, dir_prefix)): shutil.rmtree(os.path.join(rpm_dir, dir_prefix)) os.makedirs(os.path.join(rpm_dir, dir_prefix)) subprocess.call(["cp -rP `ls -a | grep -Ev '\.$|\.\.$|debian$|redhat$'` %s" % os.path.join(rpm_dir, dir_prefix)], cwd=self.builder.workdir, shell=True ) cur_dir = os.getcwd() os.chdir(rpm_dir) source_file = os.path.join(rpm_dir, 'SOURCES', '%s.tar.gz' % dir_prefix) tar = tarfile.open(source_file, 'w:gz') tar.add(dir_prefix) tar.close() shutil.rmtree(dir_prefix) os.chdir(cur_dir) if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd: self.project.install_cmd = 'cp -r \`ls -a | grep -Ev "\.$|\.\.$|debian$"\` %s/%s/%s' % ( build_dir, self.project.install_prefix, self.project.name ) template_data = { 'name': self.project.name, 'version': self.version, 'build_dir': build_dir, 'build_cmd': self.project.build_cmd, 'install_cmd': self.builder.mod_install_cmd, 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a %h %d %Y"), 'git_url': self.project.git_url, 'source': source_file, } rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False environment = None if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: rvmexec = open(rvm_rc).read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/rvm/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() rvm_env = {} for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: try: os.environ.pop('PATH') os.environ.pop('GEM_HOME') os.environ.pop('BUNDLER_PATH') except Exception, e: pass for param in os.environ.keys(): if param.find('PROXY') != -1: rvm_env[param] = os.environ[param] rvm_env.update(os.environ) environment = rvm_env log.info(environment) if os.path.isfile(os.path.join(self.builder.workdir, 'rpm', "%s.spec" % self.project.name)): self.dos2unix(os.path.join(self.builder.workdir, 'rpm', "%s.spec" % self.project.name)) template_fd = open(os.path.join(self.builder.workdir, 'rpm', "%s.spec" % self.project.name)) else: template_fd = open(os.path.join(templates_dir, 'project.spec')) rendered_template = open(spec_filename, 'w+') rendered_template.write(pystache.template.Template(template_fd.read()).render(context=template_data)) template_fd.close() rendered_template.close() rendered_template = open(spec_filename, 'a') rendered_template.write("* %(date)s %(username)s <%(email)s> - %(version)s-1\n" % template_data) for git_log in self.builder.git.log(): rendered_template.write('- %s' % git_log) rendered_template.close() self.project.save() if type(environment) is NoneType: environment = os.environ rpm_cmd = self.builder._exec([ "rpmbuild", "--define", "_topdir %s" % rpm_dir, "-ba", spec_filename ], cwd=self.builder.workdir, env=environment, stdout=self.stdout, stderr=self.stderr ) rpm_cmd.wait() for path, dirs, files in os.walk(rpm_dir): if os.path.isdir(path): for file in (os.path.join(path, file) for file in files): try: if os.path.isfile(file) and file.endswith('.rpm'): shutil.copy(file, self.builder.workspace) except Exception, e: log.error(e)
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """A simple script to exercise the BuildInfo module""" import sys import os SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.dirname(SCRIPT_DIR)) from build_info import BuildInfo FILENAME = os.path.join(SCRIPT_DIR, "sl_test.info") INFO = BuildInfo(FILENAME) PREFIX = INFO.get("PREFIX") if not os.path.exists(PREFIX): print "PREFIX doesn't exist" os.unlink(FILENAME) sys.exit(-1) if SCRIPT_DIR not in PREFIX: print "PREFIX not at expected location" os.unlink(FILENAME) sys.exit(-1) if not os.path.exists(INFO.get("HWLOC_PREFIX")): print "No hwloc directory" os.unlink(FILENAME)
def build(self, branch, last_tag=None, force_version=None, force_release=None): templates = {} templates_dir = os.path.join(self.builder.templates_dir, 'deb') control_data_new = None self.build_info = BuildInfo(self.project.name) logfile = os.path.join( BrickConfig().get("workspace", "dir"), 'log', '%s.%s.log' % ( self.project.name, self.build_info.build_id ) ) log.info("build log file: %s" % logfile) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout self.debian_dir = os.path.join(self.builder.workdir, 'debian') if last_tag is not None: os.environ.update({'BRICKLAYER_RELEASE': last_tag.split('_')[0]}) os.environ.update({'BRICKLAYER_TAG': last_tag}) # Not now #self.build_install_deps() if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd : self.project.install_cmd = 'cp -r \`ls | grep -v debian\` debian/tmp/%s' % ( self.project.install_prefix ) changelog_entry, changelog_data = self.configure_changelog(branch) if changelog_entry and changelog_data: if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('stable'), 'branch': 'stable'}) self.build_info.version(self.project.version('stable')) self.build_info.release('stable') elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('testing'), 'branch': 'testing'}) self.build_info.version(self.project.version('testing')) self.build_info.release('testing') elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('unstable'), 'branch': 'unstable'}) self.build_info.version(self.project.version('unstable')) self.build_info.release('unstable') else: """ otherwise it should change the distribution to experimental """ version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) changelog_data.update({'version': self.project.version(branch), 'branch': 'experimental'}) self.build_info.version(self.project.version(branch)) self.build_info.release('experimental:%s' % branch) with file(os.path.join(self.builder.workdir, 'debian', 'changelog'), 'w') as fh: fh.write(changelog_entry % changelog_data) else: self.build_info.version(force_version) self.build_info.release(force_release) self.project.version(branch, force_version) self.project.version(force_release, force_version) rvm_env = {} rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: with open(rvm_rc) as tmpfh: rvmexec = tmpfh.read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/rvm/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: os_env = dict(os.environ) for k in ("PATH", "GEM_HOME", "BUNDLER_PATH"): if (k in os_env): del(os_env[k]) rvm_env.update(os_env) log.info(rvm_env) os.chmod(os.path.join(self.debian_dir, 'rules'), stat.S_IRWXU|stat.S_IRWXG|stat.S_IROTH|stat.S_IXOTH) dpkg_cmd = self.builder._exec( ['dpkg-buildpackage', '-rfakeroot', '-tc', '-k%s' % BrickConfig().get('gpg', 'keyid')], cwd=self.builder.workdir, env=rvm_env, stdout=self.stdout, stderr=self.stderr, close_fds=True) dpkg_cmd.wait() clean_cmd = self.builder._exec(['dh', 'clean'], cwd=self.builder.workdir, stdout=self.stdout, stderr=self.stderr, close_fds=True) clean_cmd.wait()
def build(self, branch, last_tag=None, force_version=None, force_release=None): templates = {} templates_dir = os.path.join(self.builder.templates_dir, 'deb') debian_dir = os.path.join(self.builder.workdir, 'debian') control_data_new = None self.build_info = BuildInfo(self.project.name) logfile = os.path.join( self.builder.workspace, 'log', '%s.%s.log' % ( self.project.name, self.build_info.build_id ) ) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd : self.project.install_cmd = 'cp -r \`ls | grep -v debian\` debian/tmp/%s' % ( self.project.install_prefix ) template_data = { 'name': self.project.name, 'version': "%s" % (self.project.version(branch)), 'build_cmd': self.project.build_cmd, 'install_cmd': self.builder.mod_install_cmd, 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } changelog = os.path.join(self.builder.workdir, 'debian', 'changelog') if hasattr(self.builder.build_options, 'changelog') and self.builder.build_options.changelog: if os.path.isfile(changelog): os.rename(changelog, "%s.save" % changelog) def read_file_data(f): template_fd = open(os.path.join(templates_dir, f)) templates[f] = pystache.template.Template(template_fd.read()).render(context=template_data) template_fd.close() if not os.path.isdir(debian_dir): map(read_file_data, ['changelog', 'control', 'rules']) os.makedirs( os.path.join( debian_dir, self.project.name, self.project.install_prefix)) for filename, data in templates.iteritems(): open(os.path.join(debian_dir, filename), 'w').write(data) changelog_entry = """%(name)s (%(version)s) %(branch)s; urgency=low * Latest commits %(commits)s -- %(username)s <%(email)s> %(date)s """ changelog_data = { 'name': self.project.name, 'version': self.project.version(branch), 'branch': branch, 'commits': ' '.join(self.builder.git.log()), 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('stable'), 'branch': 'stable'}) self.build_info.version(self.project.version('stable')) self.build_info.release('stable') elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('testing'), 'branch': 'testing'}) self.build_info.version(self.project.version('testing')) self.build_info.release('testing') elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('unstable'), 'branch': 'unstable'}) self.build_info.version(self.project.version('unstable')) self.build_info.release('unstable') else: """ otherwise it should change the distribution to experimental """ if self.project.version(branch): version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) changelog_data.update({'version': self.project.version(branch), 'branch': 'experimental'}) self.build_info.version(self.project.version(branch)) self.build_info.release('experimental:%s' % branch) open(os.path.join(self.builder.workdir, 'debian', 'changelog'), 'w').write(changelog_entry % changelog_data) else: self.build_info.version(force_version) self.build_info.release(force_release) rvm_env = {} rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: rvmexec = open(rvm_rc).read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: try: os.environ.pop('PATH') os.environ.pop('GEM_HOME') os.environ.pop('BUNDLER_PATH') except Exception, e: pass rvm_env.update(os.environ)
class BuilderDeb(): def __init__(self, builder): self.builder = builder self.project = self.builder.project log.info("Debian builder initialized: %s" % self.builder.workdir) log.info("Building [%s] with options: %s" % ( self.project.name, self.builder.build_options.options )) def configure_changelog(self, branch): template_data = { 'name': self.project.name, 'version': "%s" % (self.project.version(branch)), 'build_cmd': self.project.build_cmd, 'install_cmd': self.builder.mod_install_cmd, 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } changelog = os.path.join(self.builder.workdir, 'debian', 'changelog') if hasattr(self.builder.build_options, 'changelog') and self.builder.build_options.changelog: if os.path.isfile(changelog): os.rename(changelog, "%s.save" % changelog) def read_file_data(f): template_fd = open(os.path.join(templates_dir, f)) templates[f] = pystache.template.Template(template_fd.read()).render(context=template_data) template_fd.close() if not os.path.isdir(self.debian_dir): map(read_file_data, ['changelog', 'control', 'rules']) os.makedirs( os.path.join(self.debian_dir, self.project.name, self.project.install_prefix)) for filename, data in templates.iteritems(): open(os.path.join(self.debian_dir, filename), 'w').write(data) changelog_entry = """%(name)s (%(version)s) %(branch)s; urgency=low * Latest commits %(commits)s -- %(username)s <%(email)s> %(date)s """ changelog_data = { 'name': self.project.name, 'version': self.project.version(branch), 'branch': branch, 'commits': ' '.join(self.builder.git.log()), 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } return (changelog_entry, changelog_data) else: return (None, None) def build_install_deps(self): p = self.builder._exec(["dpkg-checkbuilddeps"], stderr=subprocess.PIPE) p.wait() out = p.stderr.read() if out != "": deps = re.findall("([a-z0-9\-]+\s|[a-z0-9\-]+$)", out.split("dependencies:")[1]) deps = map(lambda x: x.strip(), deps) apt_cmd = "apt-get -y --force-yes install %s" % " ".join(deps) b = self.builder._exec(apt_cmd.split(), stdout=self.stdout, stderr=self.stderr) b.wait() def build(self, branch, last_tag=None, force_version=None, force_release=None): templates = {} templates_dir = os.path.join(self.builder.templates_dir, 'deb') control_data_new = None self.build_info = BuildInfo(self.project.name) logfile = os.path.join( self.builder.workspace, 'log', '%s.%s.log' % ( self.project.name, self.build_info.build_id ) ) log.info("build log file: %s" % logfile) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout self.debian_dir = os.path.join(self.builder.workdir, 'debian') if last_tag is not None: os.environ.update({'BRICKLAYER_RELEASE': last_tag.split('_')[0]}) os.environ.update({'BRICKLAYER_TAG': last_tag}) # Not now #self.build_install_deps() if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd : self.project.install_cmd = 'cp -r \`ls | grep -v debian\` debian/tmp/%s' % ( self.project.install_prefix) changelog_entry, changelog_data = self.configure_changelog(branch) if changelog_entry and changelog_data: if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('stable'), 'branch': 'stable'}) self.build_info.version(self.project.version('stable')) self.build_info.release('stable') elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('testing'), 'branch': 'testing'}) self.build_info.version(self.project.version('testing')) self.build_info.release('testing') elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('unstable'), 'branch': 'unstable'}) self.build_info.version(self.project.version('unstable')) self.build_info.release('unstable') else: """ otherwise it should change the distribution to experimental """ version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) changelog_data.update({'version': self.project.version(branch), 'branch': 'experimental'}) self.build_info.version(self.project.version(branch)) self.build_info.release('experimental:%s' % branch) with file(os.path.join(self.builder.workdir, 'debian', 'changelog'), 'w') as fh: fh.write(changelog_entry % changelog_data) else: self.build_info.version(force_version) self.build_info.release(force_release) self.project.version(branch, force_version) self.project.version(force_release, force_version) rvm_env = {} rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: rvmexec = open(rvm_rc).read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/rvm/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: os_env = dict(os.environ) for k in ("PATH", "GEM_HOME", "BUNDLER_PATH"): if (k in os_env): del(os_env[k]) rvm_env.update(os_env) os.chmod(os.path.join(self.debian_dir, 'rules'), stat.S_IRWXU|stat.S_IRWXG|stat.S_IROTH|stat.S_IXOTH) dpkg_cmd = self.builder._exec( ['dpkg-buildpackage', '-rfakeroot', '-tc', '-k%s' % BrickConfig().get('gpg', 'keyid')], cwd=self.builder.workdir, env=rvm_env, stdout=self.stdout, stderr=self.stderr ) dpkg_cmd.wait() clean_cmd = self.builder._exec(['dh', 'clean'], cwd=self.builder.workdir, stdout=self.stdout, stderr=self.stderr) clean_cmd.wait() def upload(self, branch): glob_str = '%s/%s_%s_*.changes' % ( BrickConfig().get('workspace', 'dir'), self.project.name, self.project.version(branch)) changes_file = glob.glob(glob_str)[0] distribution, files = self.parse_changes(changes_file) self.local_repo(distribution, files) try: self.upload_files(distribution, files) upload_file = changes_file.replace('.changes', '.upload') open(upload_file, 'w').write("done") except Exception, e: log.error("Package could not be uploaded: %s", e)
if daos_server is None: print("No PID was found for the DAOS server") elif "No such process" in e: print("The daos_server process is no longer available" " and could not be killed.") else: print("Unable to shut down DAOS server: {}".format(e)) if __name__ == "__main__": """ Start a DAOS server and then run the four stages of the client. """ print("Running rdb tests") rc = 0 binfo = BuildInfo(os.path.join(build_root, ".build_vars.json")) debug_cmds = "-x D_LOG_MASK=DEBUG,RPC=ERR,MEM=ERR " + \ "-x DD_SUBSYS=all -x DD_MASK=all" if not load_mpi('openmpi'): raise ServerFailedToStart("No orterun installed") orterun = find_executable('orterun') if orterun is None: raise ServerFailedToStart("No orterun installed") try: # Server operations p = start_server(binfo, orterun) counter = 0 daos_server = daos_server_pid() while daos_server is None:
def get(self, project, bid): build_info = BuildInfo(project, bid) if os.path.isfile(build_info.log()): self.write(open(build_info.log()).read())
class BuilderRpm(): def __init__(self, builder): self.builder = builder self.project = self.builder.project self.distribution = None self.version = None def dos2unix(self, file): f = open(file, 'r').readlines() new_file = open(file, "w+") match = re.compile('\r\n') for line in f: new_file.write(match.sub('\n', line)) new_file.close() def build(self, branch, last_tag=None): templates_dir = os.path.join(self.builder.templates_dir, 'rpm') rpm_dir = os.path.join(self.builder.workdir, 'redhat') spec_filename = os.path.join(rpm_dir, 'SPECS', "%s.spec" % self.project.name) self.build_info = BuildInfo(self.project.name) logfile = os.path.join( self.builder.workspace, 'log', '%s.%s.log' % (self.project.name, self.build_info.build_id)) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) self.build_info.version(self.project.version('stable')) self.version = self.project.version('stable') self.distribution = 'stable' elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) self.build_info.version(self.project.version('testing')) self.version = self.project.version('testing') self.distribution = 'testing' elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) self.build_info.version(self.project.version('unstable')) self.version = self.project.version('unstable') self.distribution = 'unstable' else: """ otherwise it should change the distribution to unstable """ if self.project.version(branch): version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str( int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) self.build_info.version(self.project.version(branch)) self.version = self.project.version(branch) self.distribution = 'experimental' dir_prefix = "%s-%s" % (self.project.name, self.version) for dir in ('SOURCES', 'SPECS', 'RPMS', 'SRPMS', 'BUILD', 'TMP'): if os.path.isdir(os.path.join(rpm_dir, dir)): shutil.rmtree(os.path.join(rpm_dir, dir)) os.makedirs(os.path.join(rpm_dir, dir)) build_dir = os.path.join(rpm_dir, 'TMP', self.project.name) os.makedirs(build_dir) if os.path.isdir(os.path.join(rpm_dir, dir_prefix)): shutil.rmtree(os.path.join(rpm_dir, dir_prefix)) os.makedirs(os.path.join(rpm_dir, dir_prefix)) subprocess.call([ "cp -rP `ls -a | grep -Ev '\.$|\.\.$|debian$|redhat$'` %s" % os.path.join(rpm_dir, dir_prefix) ], cwd=self.builder.workdir, shell=True) cur_dir = os.getcwd() os.chdir(rpm_dir) source_file = os.path.join(rpm_dir, 'SOURCES', '%s.tar.gz' % dir_prefix) tar = tarfile.open(source_file, 'w:gz') tar.add(dir_prefix) tar.close() shutil.rmtree(dir_prefix) os.chdir(cur_dir) if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd: self.project.install_cmd = 'cp -r \`ls -a | grep -Ev "\.$|\.\.$|debian$"\` %s/%s/%s' % ( build_dir, self.project.install_prefix, self.project.name) template_data = { 'name': self.project.name, 'version': self.version, 'build_dir': build_dir, 'build_cmd': self.project.build_cmd, 'install_cmd': self.builder.mod_install_cmd, 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a %h %d %Y"), 'git_url': self.project.git_url, 'source': source_file, } rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False environment = None if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: rvmexec = open(rvm_rc).read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/rvm/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() rvm_env = {} for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: try: os.environ.pop('PATH') os.environ.pop('GEM_HOME') os.environ.pop('BUNDLER_PATH') except Exception, e: pass for param in os.environ.keys(): if param.find('PROXY') != -1: rvm_env[param] = os.environ[param] rvm_env.update(os.environ) environment = rvm_env log.info(environment) if os.path.isfile( os.path.join(self.builder.workdir, 'rpm', "%s.spec" % self.project.name)): self.dos2unix( os.path.join(self.builder.workdir, 'rpm', "%s.spec" % self.project.name)) template_fd = open( os.path.join(self.builder.workdir, 'rpm', "%s.spec" % self.project.name)) else: template_fd = open(os.path.join(templates_dir, 'project.spec')) rendered_template = open(spec_filename, 'w+') rendered_template.write( pystache.template.Template( template_fd.read()).render(context=template_data)) template_fd.close() rendered_template.close() rendered_template = open(spec_filename, 'a') rendered_template.write( "* %(date)s %(username)s <%(email)s> - %(version)s-1\n" % template_data) for git_log in self.builder.git.log(): rendered_template.write('- %s' % git_log) rendered_template.close() self.project.save() if type(environment) is NoneType: environment = os.environ rpm_cmd = self.builder._exec([ "rpmbuild", "--define", "_topdir %s" % rpm_dir, "-ba", spec_filename ], cwd=self.builder.workdir, env=environment, stdout=self.stdout, stderr=self.stderr) rpm_cmd.wait() for path, dirs, files in os.walk(rpm_dir): if os.path.isdir(path): for file in (os.path.join(path, file) for file in files): try: if os.path.isfile(file) and file.endswith('.rpm'): shutil.copy(file, self.builder.workspace) except Exception, e: log.error(e)
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. """A simple script to exercise the BuildInfo module""" from __future__ import print_function import sys import os SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.dirname(SCRIPT_DIR)) from build_info import BuildInfo FILENAME = os.path.join(SCRIPT_DIR, "sl_test.info") INFO = BuildInfo(FILENAME) PREFIX = INFO.get("PREFIX") if not os.path.exists(PREFIX): print("PREFIX doesn't exist") os.unlink(FILENAME) sys.exit(-1) if SCRIPT_DIR not in PREFIX: print("PREFIX not at expected location") os.unlink(FILENAME) sys.exit(-1) SH_SCRIPT = os.path.join(SCRIPT_DIR, "sl_test.sh") INFO.gen_script(SH_SCRIPT) os.system("source %s" % SH_SCRIPT)
def build(self, branch, last_tag=None): templates = {} templates_dir = os.path.join(self.builder.templates_dir, 'deb') debian_dir = os.path.join(self.builder.workdir, 'debian') control_data_new = None self.build_info = BuildInfo(self.project.name) logfile = os.path.join(self.builder.workspace, 'log', '%s.%s.log' % (self.project.name, self.build_info.build_id)) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd : self.project.install_cmd = 'cp -r \`ls | grep -v debian\` debian/tmp/%s' % ( self.project.install_prefix ) template_data = { 'name': self.project.name, 'version': "%s" % (self.project.version(branch)), 'build_cmd': self.project.build_cmd, 'install_cmd': self.builder.mod_install_cmd, 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } changelog = os.path.join(self.builder.workdir, 'debian', 'changelog') if os.path.isfile(changelog): os.rename(changelog, "%s.save" % changelog) def read_file_data(f): template_fd = open(os.path.join(templates_dir, f)) templates[f] = pystache.template.Template(template_fd.read()).render(context=template_data) template_fd.close() if not os.path.isdir(debian_dir): map(read_file_data, ['changelog', 'control', 'rules']) os.makedirs( os.path.join( debian_dir, self.project.name, self.project.install_prefix)) for filename, data in templates.iteritems(): open(os.path.join(debian_dir, filename), 'w').write(data) changelog_entry = """%(name)s (%(version)s) %(branch)s; urgency=low * Latest commits %(commits)s -- %(username)s <%(email)s> %(date)s """ changelog_data = { 'name': self.project.name, 'version': self.project.version(branch), 'branch': branch, 'commits': ' '.join(self.builder.git.log()), 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('stable'), 'branch': 'stable'}) self.build_info.version(self.project.version('stable')) self.build_info.release('stable') elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('testing'), 'branch': 'testing'}) self.build_info.version(self.project.version('testing')) self.build_info.release('testing') elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('unstable'), 'branch': 'unstable'}) self.build_info.version(self.project.version('unstable')) self.build_info.release('unstable') else: """ otherwise it should change the distribution to experimental """ if self.project.version(branch): version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) changelog_data.update({'version': self.project.version(branch), 'branch': 'experimental'}) self.build_info.version(self.project.version(branch)) self.build_info.release('experimental:%s' % branch) open(os.path.join(self.builder.workdir, 'debian', 'changelog'), 'w').write(changelog_entry % changelog_data) rvm_env = {} rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: rvmexec = open(rvm_rc).read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: try: os.environ.pop('PATH') os.environ.pop('GEM_HOME') os.environ.pop('BUNDLER_PATH') except Exception, e: pass rvm_env.update(os.environ)
class BuilderDeb: def __init__(self, builder): self.builder = builder self.project = self.builder.project log.msg("Building [%s] with options: %s" % (self.project.name, self.builder.build_options.options)) def configure_changelog(self, branch): template_data = { "name": self.project.name, "version": "%s" % (self.project.version(branch)), "build_cmd": self.project.build_cmd, "install_cmd": self.builder.mod_install_cmd, "username": self.project.username, "email": self.project.email, "date": time.strftime("%a, %d %h %Y %T %z"), } changelog = os.path.join(self.builder.workdir, "debian", "changelog") if hasattr(self.builder.build_options, "changelog") and self.builder.build_options.changelog: if os.path.isfile(changelog): os.rename(changelog, "%s.save" % changelog) def read_file_data(f): with open(os.path.join(templates_dir, f)) as tmpfh: templates[f] = pystache.template.Template(tmpfh.read()).render(context=template_data) if not os.path.isdir(self.debian_dir): map(read_file_data, ["changelog", "control", "rules"]) os.makedirs(os.path.join(self.debian_dir, self.project.name, self.project.install_prefix)) for filename, data in templates.iteritems(): with open(os.path.join(self.debian_dir, filename), "w") as tmpfh: tmpfh.write(data) changelog_entry = """%(name)s (%(version)s) %(branch)s; urgency=low * Latest commits %(commits)s -- %(username)s <%(email)s> %(date)s """ changelog_data = { "name": self.project.name, "version": self.project.version(branch), "branch": branch, "commits": " ".join(self.builder.git.log()), "username": self.project.username, "email": self.project.email, "date": time.strftime("%a, %d %h %Y %T %z"), } return (changelog_entry, changelog_data) else: return (None, None) def build_install_deps(self): p = self.builder._exec(["dpkg-checkbuilddeps"], stderr=subprocess.PIPE, close_fds=True) p.wait() out = p.stderr.read() if out != "": deps = re.findall("([a-z0-9\-]+\s|[a-z0-9\-]+$)", out.split("dependencies:")[1]) deps = map(lambda x: x.strip(), deps) apt_cmd = "apt-get -y --force-yes install %s" % " ".join(deps) b = self.builder._exec(apt_cmd.split(), stdout=self.stdout, stderr=self.stderr, close_fds=True) b.wait() def build(self, branch, last_tag=None, force_version=None, force_release=None): templates = {} templates_dir = os.path.join(self.builder.templates_dir, "deb") control_data_new = None self.build_info = BuildInfo(self.project.name) logfile = os.path.join( BrickConfig().get("workspace", "dir"), "log", "%s.%s.log" % (self.project.name, self.build_info.build_id) ) self.build_info.log(logfile) self.stdout = open(logfile, "a+") self.stderr = self.stdout self.debian_dir = os.path.join(self.builder.workdir, "debian") if last_tag is not None: os.environ.update({"BRICKLAYER_RELEASE": last_tag.split("_")[0]}) os.environ.update({"BRICKLAYER_TAG": last_tag}) # Not now # self.build_install_deps() if self.project.install_prefix is None: self.project.install_prefix = "opt" if not self.project.install_cmd: self.project.install_cmd = "cp -r \`ls | grep -v debian\` debian/tmp/%s" % (self.project.install_prefix) changelog_entry, changelog_data = self.configure_changelog(branch) if changelog_entry and changelog_data: if last_tag != None and last_tag.startswith("stable"): self.project.version("stable", last_tag.split("_")[1]) changelog_data.update({"version": self.project.version("stable"), "branch": "stable"}) self.build_info.version(self.project.version("stable")) self.build_info.release("stable") elif last_tag != None and last_tag.startswith("testing"): self.project.version("testing", last_tag.split("_")[1]) changelog_data.update({"version": self.project.version("testing"), "branch": "testing"}) self.build_info.version(self.project.version("testing")) self.build_info.release("testing") elif last_tag != None and last_tag.startswith("unstable"): self.project.version("unstable", last_tag.split("_")[1]) changelog_data.update({"version": self.project.version("unstable"), "branch": "unstable"}) self.build_info.version(self.project.version("unstable")) self.build_info.release("unstable") else: """ otherwise it should change the distribution to experimental """ version_list = self.project.version(branch).split(".") version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, ".".join(version_list)) changelog_data.update({"version": self.project.version(branch), "branch": "experimental"}) self.build_info.version(self.project.version(branch)) self.build_info.release("experimental:%s" % branch) with file(os.path.join(self.builder.workdir, "debian", "changelog"), "w") as fh: fh.write(changelog_entry % changelog_data) else: self.build_info.version(force_version) self.build_info.release(force_release) self.project.version(branch, force_version) self.project.version(force_release, force_version) rvm_rc = os.path.join(self.builder.workdir, ".rvmrc") rvm_rc_example = rvm_rc + ".example" has_rvm = False has_rvm = os.path.isfile(rvm_rc) or os.path.isfile(rvm_rc_example) os.chmod(os.path.join(self.debian_dir, "rules"), stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH) if has_rvm: dpkg_cmd = self.builder._exec( [ "/usr/bin/bricklayer-rvm-exec", "/usr/bin/dpkg-buildpackage", "-rfakeroot", "-tc", "-k%s" % BrickConfig().get("gpg", "keyid"), ], cwd=self.builder.workdir, stdout=self.stdout, stderr=self.stderr, close_fds=True, ) else: dpkg_cmd = self.builder._exec( ["dpkg-buildpackage", "-rfakeroot", "-tc", "-k%s" % BrickConfig().get("gpg", "keyid")], cwd=self.builder.workdir, stdout=self.stdout, stderr=self.stderr, close_fds=True, ) dpkg_cmd.wait() clean_cmd = self.builder._exec( ["dh", "clean"], cwd=self.builder.workdir, stdout=self.stdout, stderr=self.stderr, close_fds=True ) clean_cmd.wait() def upload(self, branch): glob_str = "%s/%s_%s_*.changes" % ( BrickConfig().get("workspace", "dir"), self.project.name, self.project.version(branch), ) changes_file = glob.glob(glob_str) distribution, files = self.parse_changes(changes_file[0]) try: self.upload_files(distribution, files) upload_file = changes_file[0].replace(".changes", ".upload") with open(upload_file, "w") as tmpfh: tmpfh.write("done") except Exception, e: log.msg("[%s] %s" % (self.project.name, traceback.format_exc()))
def get(self, project, build): build_info = BuildInfo(project, build) if os.path.isfile(build_info.log()): self.write(open(build_info.log()).read())
def build(self, branch, last_tag=None, force_version=None, force_release=None): templates = {} templates_dir = os.path.join(self.builder.templates_dir, "deb") control_data_new = None self.build_info = BuildInfo(self.project.name) logfile = os.path.join( BrickConfig().get("workspace", "dir"), "log", "%s.%s.log" % (self.project.name, self.build_info.build_id) ) self.build_info.log(logfile) self.stdout = open(logfile, "a+") self.stderr = self.stdout self.debian_dir = os.path.join(self.builder.workdir, "debian") if last_tag is not None: os.environ.update({"BRICKLAYER_RELEASE": last_tag.split("_")[0]}) os.environ.update({"BRICKLAYER_TAG": last_tag}) # Not now # self.build_install_deps() if self.project.install_prefix is None: self.project.install_prefix = "opt" if not self.project.install_cmd: self.project.install_cmd = "cp -r \`ls | grep -v debian\` debian/tmp/%s" % (self.project.install_prefix) changelog_entry, changelog_data = self.configure_changelog(branch) if changelog_entry and changelog_data: if last_tag != None and last_tag.startswith("stable"): self.project.version("stable", last_tag.split("_")[1]) changelog_data.update({"version": self.project.version("stable"), "branch": "stable"}) self.build_info.version(self.project.version("stable")) self.build_info.release("stable") elif last_tag != None and last_tag.startswith("testing"): self.project.version("testing", last_tag.split("_")[1]) changelog_data.update({"version": self.project.version("testing"), "branch": "testing"}) self.build_info.version(self.project.version("testing")) self.build_info.release("testing") elif last_tag != None and last_tag.startswith("unstable"): self.project.version("unstable", last_tag.split("_")[1]) changelog_data.update({"version": self.project.version("unstable"), "branch": "unstable"}) self.build_info.version(self.project.version("unstable")) self.build_info.release("unstable") else: """ otherwise it should change the distribution to experimental """ version_list = self.project.version(branch).split(".") version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, ".".join(version_list)) changelog_data.update({"version": self.project.version(branch), "branch": "experimental"}) self.build_info.version(self.project.version(branch)) self.build_info.release("experimental:%s" % branch) with file(os.path.join(self.builder.workdir, "debian", "changelog"), "w") as fh: fh.write(changelog_entry % changelog_data) else: self.build_info.version(force_version) self.build_info.release(force_release) self.project.version(branch, force_version) self.project.version(force_release, force_version) rvm_rc = os.path.join(self.builder.workdir, ".rvmrc") rvm_rc_example = rvm_rc + ".example" has_rvm = False has_rvm = os.path.isfile(rvm_rc) or os.path.isfile(rvm_rc_example) os.chmod(os.path.join(self.debian_dir, "rules"), stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH) if has_rvm: dpkg_cmd = self.builder._exec( [ "/usr/bin/bricklayer-rvm-exec", "/usr/bin/dpkg-buildpackage", "-rfakeroot", "-tc", "-k%s" % BrickConfig().get("gpg", "keyid"), ], cwd=self.builder.workdir, stdout=self.stdout, stderr=self.stderr, close_fds=True, ) else: dpkg_cmd = self.builder._exec( ["dpkg-buildpackage", "-rfakeroot", "-tc", "-k%s" % BrickConfig().get("gpg", "keyid")], cwd=self.builder.workdir, stdout=self.stdout, stderr=self.stderr, close_fds=True, ) dpkg_cmd.wait() clean_cmd = self.builder._exec( ["dh", "clean"], cwd=self.builder.workdir, stdout=self.stdout, stderr=self.stderr, close_fds=True ) clean_cmd.wait()
except Exception as e: if daos_server is None: print("No PID was found for the DAOS server") elif "No such process" in e: print("The daos_server process is no longer available" " and could not be killed.") else: print("Unable to shut down DAOS server: {}".format(e)) if __name__ == "__main__": """ Start a DAOS server and then run the four stages of the client. """ print("Running rdb tests") rc = 0 binfo = BuildInfo(os.path.join(build_root, ".build_vars.json")); debug_cmds = "-x D_LOG_MASK=DEBUG,RPC=ERR,MEM=ERR " + \ "-x DD_SUBSYS=all -x DD_MASK=all" try: # Server operations p = start_server(binfo) counter = 0 daos_server = daos_server_pid() while daos_server is None: if counter >= 120: raise ServerTimedOut("No DAOS server process detected before "\ "timeout") counter += 1 time.sleep(1)
class BuilderDeb(): def __init__(self, builder): self.builder = builder self.project = self.builder.project log.info("Debian builder initialized: %s" % self.builder.workdir) log.info("Building [%s] with options: %s" % ( self.project.name, self.builder.build_options.options )) def configure_changelog(self, branch): template_data = { 'name': self.project.name, 'version': "%s" % (self.project.version(branch)), 'build_cmd': self.project.build_cmd, 'install_cmd': self.builder.mod_install_cmd, 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } changelog = os.path.join(self.builder.workdir, 'debian', 'changelog') if hasattr(self.builder.build_options, 'changelog') and self.builder.build_options.changelog: if os.path.isfile(changelog): os.rename(changelog, "%s.save" % changelog) def read_file_data(f): with open(os.path.join(templates_dir, f)) as tmpfh: templates[f] = pystache.template.Template(tmpfh.read()).render(context=template_data) if not os.path.isdir(self.debian_dir): map(read_file_data, ['changelog', 'control', 'rules']) os.makedirs( os.path.join(self.debian_dir, self.project.name, self.project.install_prefix)) for filename, data in templates.iteritems(): with open(os.path.join(self.debian_dir, filename), 'w') as tmpfh: tmpfh.write(data) changelog_entry = """%(name)s (%(version)s) %(branch)s; urgency=low * Latest commits %(commits)s -- %(username)s <%(email)s> %(date)s """ changelog_data = { 'name': self.project.name, 'version': self.project.version(branch), 'branch': branch, 'commits': ' '.join(self.builder.git.log()), 'username': self.project.username, 'email': self.project.email, 'date': time.strftime("%a, %d %h %Y %T %z"), } return (changelog_entry, changelog_data) else: return (None, None) def build_install_deps(self): p = self.builder._exec(["dpkg-checkbuilddeps"], stderr=subprocess.PIPE, close_fds=True) p.wait() out = p.stderr.read() if out != "": deps = re.findall("([a-z0-9\-]+\s|[a-z0-9\-]+$)", out.split("dependencies:")[1]) deps = map(lambda x: x.strip(), deps) apt_cmd = "apt-get -y --force-yes install %s" % " ".join(deps) b = self.builder._exec(apt_cmd.split(), stdout=self.stdout, stderr=self.stderr, close_fds=True) b.wait() def build(self, branch, last_tag=None, force_version=None, force_release=None): templates = {} templates_dir = os.path.join(self.builder.templates_dir, 'deb') control_data_new = None self.build_info = BuildInfo(self.project.name) logfile = os.path.join( BrickConfig().get("workspace", "dir"), 'log', '%s.%s.log' % ( self.project.name, self.build_info.build_id ) ) log.info("build log file: %s" % logfile) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout self.debian_dir = os.path.join(self.builder.workdir, 'debian') if last_tag is not None: os.environ.update({'BRICKLAYER_RELEASE': last_tag.split('_')[0]}) os.environ.update({'BRICKLAYER_TAG': last_tag}) # Not now #self.build_install_deps() if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd : self.project.install_cmd = 'cp -r \`ls | grep -v debian\` debian/tmp/%s' % ( self.project.install_prefix ) changelog_entry, changelog_data = self.configure_changelog(branch) if changelog_entry and changelog_data: if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('stable'), 'branch': 'stable'}) self.build_info.version(self.project.version('stable')) self.build_info.release('stable') elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('testing'), 'branch': 'testing'}) self.build_info.version(self.project.version('testing')) self.build_info.release('testing') elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('unstable'), 'branch': 'unstable'}) self.build_info.version(self.project.version('unstable')) self.build_info.release('unstable') else: """ otherwise it should change the distribution to experimental """ version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) changelog_data.update({'version': self.project.version(branch), 'branch': 'experimental'}) self.build_info.version(self.project.version(branch)) self.build_info.release('experimental:%s' % branch) with file(os.path.join(self.builder.workdir, 'debian', 'changelog'), 'w') as fh: fh.write(changelog_entry % changelog_data) else: self.build_info.version(force_version) self.build_info.release(force_release) self.project.version(branch, force_version) self.project.version(force_release, force_version) rvm_env = {} rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: with open(rvm_rc) as tmpfh: rvmexec = tmpfh.read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/rvm/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: os_env = dict(os.environ) for k in ("PATH", "GEM_HOME", "BUNDLER_PATH"): if (k in os_env): del(os_env[k]) rvm_env.update(os_env) log.info(rvm_env) os.chmod(os.path.join(self.debian_dir, 'rules'), stat.S_IRWXU|stat.S_IRWXG|stat.S_IROTH|stat.S_IXOTH) dpkg_cmd = self.builder._exec( ['dpkg-buildpackage', '-rfakeroot', '-tc', '-k%s' % BrickConfig().get('gpg', 'keyid')], cwd=self.builder.workdir, env=rvm_env, stdout=self.stdout, stderr=self.stderr, close_fds=True) dpkg_cmd.wait() clean_cmd = self.builder._exec(['dh', 'clean'], cwd=self.builder.workdir, stdout=self.stdout, stderr=self.stderr, close_fds=True) clean_cmd.wait() def upload(self, branch): glob_str = '%s/%s_%s_*.changes' % ( BrickConfig().get('workspace', 'dir'), self.project.name, self.project.version(branch)) changes_file = glob.glob(glob_str) log.info(changes_file) distribution, files = self.parse_changes(changes_file[0]) try: self.upload_files(distribution, files) upload_file = changes_file[0].replace('.changes', '.upload') with open(upload_file, 'w') as tmpfh: tmpfh.write("done") except Exception, e: log.error("Package could not be uploaded: %s", e)
def build(self, branch, last_tag=None, force_version=None, force_release=None): templates = {} templates_dir = os.path.join(self.builder.templates_dir, 'deb') control_data_new = None self.build_info = BuildInfo(self.project.name) logfile = os.path.join( self.builder.workspace, 'log', '%s.%s.log' % ( self.project.name, self.build_info.build_id ) ) log.info("build log file: %s" % logfile) self.build_info.log(logfile) self.stdout = open(logfile, 'a+') self.stderr = self.stdout self.debian_dir = os.path.join(self.builder.workdir, 'debian') if last_tag is not None: os.environ.update({'BRICKLAYER_RELEASE': last_tag.split('_')[0]}) os.environ.update({'BRICKLAYER_TAG': last_tag}) # Not now #self.build_install_deps() if self.project.install_prefix is None: self.project.install_prefix = 'opt' if not self.project.install_cmd : self.project.install_cmd = 'cp -r \`ls | grep -v debian\` debian/tmp/%s' % ( self.project.install_prefix) changelog_entry, changelog_data = self.configure_changelog(branch) if changelog_entry and changelog_data: if last_tag != None and last_tag.startswith('stable'): self.project.version('stable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('stable'), 'branch': 'stable'}) self.build_info.version(self.project.version('stable')) self.build_info.release('stable') elif last_tag != None and last_tag.startswith('testing'): self.project.version('testing', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('testing'), 'branch': 'testing'}) self.build_info.version(self.project.version('testing')) self.build_info.release('testing') elif last_tag != None and last_tag.startswith('unstable'): self.project.version('unstable', last_tag.split('_')[1]) changelog_data.update({'version': self.project.version('unstable'), 'branch': 'unstable'}) self.build_info.version(self.project.version('unstable')) self.build_info.release('unstable') else: """ otherwise it should change the distribution to experimental """ version_list = self.project.version(branch).split('.') version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1) self.project.version(branch, '.'.join(version_list)) changelog_data.update({'version': self.project.version(branch), 'branch': 'experimental'}) self.build_info.version(self.project.version(branch)) self.build_info.release('experimental:%s' % branch) with file(os.path.join(self.builder.workdir, 'debian', 'changelog'), 'w') as fh: fh.write(changelog_entry % changelog_data) else: self.build_info.version(force_version) self.build_info.release(force_release) self.project.version(branch, force_version) self.project.version(force_release, force_version) rvm_env = {} rvm_rc = os.path.join(self.builder.workdir, '.rvmrc') rvm_rc_example = rvm_rc + ".example" has_rvm = False if os.path.isfile(rvm_rc): has_rvm = True elif os.path.isfile(rvm_rc_example): has_rvm = True rvm_rc = rvm_rc_example if has_rvm: rvmexec = open(rvm_rc).read() log.info("RVMRC: %s" % rvmexec) # I need the output not to log on file rvm_cmd = subprocess.Popen('/usr/local/rvm/bin/rvm info %s' % rvmexec.split()[1], shell=True, stdout=subprocess.PIPE) rvm_cmd.wait() for line in rvm_cmd.stdout.readlines(): if 'PATH' in line or 'HOME' in line: name, value = line.split() rvm_env[name.strip(':')] = value.strip('"') rvm_env['HOME'] = os.environ['HOME'] if len(rvm_env.keys()) < 1: rvm_env = os.environ else: os_env = dict(os.environ) for k in ("PATH", "GEM_HOME", "BUNDLER_PATH"): if (k in os_env): del(os_env[k]) rvm_env.update(os_env) os.chmod(os.path.join(self.debian_dir, 'rules'), stat.S_IRWXU|stat.S_IRWXG|stat.S_IROTH|stat.S_IXOTH) dpkg_cmd = self.builder._exec( ['dpkg-buildpackage', '-rfakeroot', '-tc', '-k%s' % BrickConfig().get('gpg', 'keyid')], cwd=self.builder.workdir, env=rvm_env, stdout=self.stdout, stderr=self.stderr ) dpkg_cmd.wait() clean_cmd = self.builder._exec(['dh', 'clean'], cwd=self.builder.workdir, stdout=self.stdout, stderr=self.stderr) clean_cmd.wait()
except Exception as e: if daos_server is None: print("No PID was found for the DAOS server") elif "No such process" in e: print("The daos_server process is no longer available" " and could not be killed.") else: print("Unable to shut down DAOS server: {}".format(e)) if __name__ == "__main__": """ Start a DAOS server and then run the four stages of the client. """ print("Running rdb tests") rc = 0 binfo = BuildInfo(os.path.join(build_root, ".build_vars.json")); try: # Server operations p = start_server(binfo) counter = 0 daos_server = daos_server_pid() while daos_server is None: if counter >= 120: raise ServerTimedOut("No DAOS server process detected before "\ "timeout") counter += 1 time.sleep(1) daos_server = daos_server_pid()