class Builder(object): def __init__(self, jenkins_url, jenkins_user, jenkins_password, config=None, ignore_cache=False, flush_cache=False, plugins_list=None): self.jenkins = Jenkins(jenkins_url, jenkins_user, jenkins_password) self.cache = CacheStorage(jenkins_url, flush=flush_cache) self.global_config = config self.ignore_cache = ignore_cache self._plugins_list = plugins_list @property def plugins_list(self): if self._plugins_list is None: self._plugins_list = self.jenkins.get_plugins_info() return self._plugins_list def load_files(self, fn): self.parser = YamlParser(self.global_config, self.plugins_list) # handle deprecated behavior if not hasattr(fn, '__iter__'): logger.warning( 'Passing single elements for the `fn` argument in ' 'Builder.load_files is deprecated. Please update your code ' 'to use a list as support for automatic conversion will be ' 'removed in a future version.') fn = [fn] files_to_process = [] for path in fn: if os.path.isdir(path): files_to_process.extend([os.path.join(path, f) for f in os.listdir(path) if (f.endswith('.yml') or f.endswith('.yaml'))]) else: files_to_process.append(path) # symlinks used to allow loading of sub-dirs can result in duplicate # definitions of macros and templates when loading all from top-level unique_files = [] for f in files_to_process: rpf = os.path.realpath(f) if rpf not in unique_files: unique_files.append(rpf) else: logger.warning("File '%s' already added as '%s', ignoring " "reference to avoid duplicating yaml " "definitions." % (f, rpf)) for in_file in unique_files: # use of ask-for-permissions instead of ask-for-forgiveness # performs better when low use cases. if hasattr(in_file, 'name'): fname = in_file.name else: fname = in_file logger.debug("Parsing YAML file {0}".format(fname)) if hasattr(in_file, 'read'): self.parser.parse_fp(in_file) else: self.parser.parse(in_file) def delete_old_managed(self, keep=None): jobs = self.jenkins.get_jobs() deleted_jobs = 0 if keep is None: keep = [job.name for job in self.parser.xml_jobs] for job in jobs: if job['name'] not in keep and \ self.jenkins.is_managed(job['name']): logger.info("Removing obsolete jenkins job {0}" .format(job['name'])) self.delete_job(job['name']) deleted_jobs += 1 else: logger.debug("Ignoring unmanaged jenkins job %s", job['name']) return deleted_jobs def delete_job(self, jobs_glob, fn=None): if fn: self.load_files(fn) self.parser.expandYaml([jobs_glob]) jobs = [j['name'] for j in self.parser.jobs] else: jobs = [jobs_glob] if jobs is not None: logger.info("Removing jenkins job(s): %s" % ", ".join(jobs)) for job in jobs: self.jenkins.delete_job(job) if(self.cache.is_cached(job)): self.cache.set(job, '') def delete_all_jobs(self): jobs = self.jenkins.get_jobs() logger.info("Number of jobs to delete: %d", len(jobs)) for job in jobs: self.delete_job(job['name']) def update_job(self, input_fn, jobs_glob=None, output=None): self.load_files(input_fn) self.parser.expandYaml(jobs_glob) self.parser.generateXML() logger.info("Number of jobs generated: %d", len(self.parser.xml_jobs)) self.parser.xml_jobs.sort(key=operator.attrgetter('name')) if (output and not hasattr(output, 'write') and not os.path.isdir(output)): logger.info("Creating directory %s" % output) try: os.makedirs(output) except OSError: if not os.path.isdir(output): raise updated_jobs = 0 for job in self.parser.xml_jobs: if output: if hasattr(output, 'write'): # `output` is a file-like object logger.info("Job name: %s", job.name) logger.debug("Writing XML to '{0}'".format(output)) try: output.write(job.output()) except IOError as exc: if exc.errno == errno.EPIPE: # EPIPE could happen if piping output to something # that doesn't read the whole input (e.g.: the UNIX # `head` command) return raise continue output_fn = os.path.join(output, job.name) logger.debug("Writing XML to '{0}'".format(output_fn)) with io.open(output_fn, 'w', encoding='utf-8') as f: f.write(job.output().decode('utf-8')) continue md5 = job.md5() if (self.jenkins.is_job(job.name) and not self.cache.is_cached(job.name)): old_md5 = self.jenkins.get_job_md5(job.name) self.cache.set(job.name, old_md5) if self.cache.has_changed(job.name, md5) or self.ignore_cache: self.jenkins.update_job(job.name, job.output()) updated_jobs += 1 self.cache.set(job.name, md5) else: logger.debug("'{0}' has not changed".format(job.name)) return self.parser.xml_jobs, updated_jobs
class Builder(object): def __init__(self, jenkins_url, jenkins_user, jenkins_password, config=None, jenkins_timeout=_DEFAULT_TIMEOUT, ignore_cache=False, flush_cache=False, plugins_list=None): self.jenkins = Jenkins(jenkins_url, jenkins_user, jenkins_password, jenkins_timeout) self.cache = CacheStorage(jenkins_url, flush=flush_cache) self.global_config = config self.ignore_cache = ignore_cache self._plugins_list = plugins_list @property def plugins_list(self): if self._plugins_list is None: self._plugins_list = self.jenkins.get_plugins_info() return self._plugins_list def load_files(self, fn): self.parser = YamlParser(self.global_config, self.plugins_list) # handle deprecated behavior, and check that it's not a file like # object as these may implement the '__iter__' attribute. if not hasattr(fn, '__iter__') or hasattr(fn, 'read'): logger.warning( 'Passing single elements for the `fn` argument in ' 'Builder.load_files is deprecated. Please update your code ' 'to use a list as support for automatic conversion will be ' 'removed in a future version.') fn = [fn] files_to_process = [] for path in fn: if not hasattr(path, 'read') and os.path.isdir(path): files_to_process.extend([os.path.join(path, f) for f in os.listdir(path) if (f.endswith('.yml') or f.endswith('.yaml'))]) else: files_to_process.append(path) # symlinks used to allow loading of sub-dirs can result in duplicate # definitions of macros and templates when loading all from top-level unique_files = [] for f in files_to_process: if hasattr(f, 'read'): unique_files.append(f) continue rpf = os.path.realpath(f) if rpf not in unique_files: unique_files.append(rpf) else: logger.warning("File '%s' already added as '%s', ignoring " "reference to avoid duplicating yaml " "definitions." % (f, rpf)) for in_file in unique_files: # use of ask-for-permissions instead of ask-for-forgiveness # performs better when low use cases. if hasattr(in_file, 'name'): fname = in_file.name else: fname = in_file logger.debug("Parsing YAML file {0}".format(fname)) if hasattr(in_file, 'read'): self.parser.parse_fp(in_file) else: self.parser.parse(in_file) def delete_old_managed(self, keep=None): jobs = self.jenkins.get_jobs() deleted_jobs = 0 if keep is None: keep = [job.name for job in self.parser.xml_jobs] for job in jobs: if job['name'] not in keep: if self.jenkins.is_managed(job['name']): logger.info("Removing obsolete jenkins job {0}" .format(job['name'])) self.delete_job(job['name']) deleted_jobs += 1 else: logger.info("Not deleting unmanaged jenkins job %s", job['name']) else: logger.debug("Keeping job %s", job['name']) return deleted_jobs def delete_job(self, jobs_glob, fn=None): if fn: self.load_files(fn) self.parser.expandYaml([jobs_glob]) jobs = [j['name'] for j in self.parser.jobs] else: jobs = [jobs_glob] if jobs is not None: logger.info("Removing jenkins job(s): %s" % ", ".join(jobs)) for job in jobs: self.jenkins.delete_job(job) if(self.cache.is_cached(job)): self.cache.set(job, '') self.cache.save() def delete_all_jobs(self): jobs = self.jenkins.get_jobs() logger.info("Number of jobs to delete: %d", len(jobs)) self.jenkins.delete_all_jobs() # Need to clear the JJB cache after deletion self.cache.clear() @parallelize def changed(self, job): md5 = job.md5() changed = self.ignore_cache or self.cache.has_changed(job.name, md5) if not changed: logger.debug("'{0}' has not changed".format(job.name)) return changed def update_jobs(self, input_fn, jobs_glob=None, output=None, n_workers=None): orig = time.time() self.load_files(input_fn) self.parser.expandYaml(jobs_glob) self.parser.generateXML() step = time.time() logging.debug('%d XML files generated in %ss', len(self.parser.jobs), str(step - orig)) logger.info("Number of jobs generated: %d", len(self.parser.xml_jobs)) self.parser.xml_jobs.sort(key=operator.attrgetter('name')) if (output and not hasattr(output, 'write') and not os.path.isdir(output)): logger.info("Creating directory %s" % output) try: os.makedirs(output) except OSError: if not os.path.isdir(output): raise if output: for job in self.parser.xml_jobs: if hasattr(output, 'write'): # `output` is a file-like object logger.info("Job name: %s", job.name) logger.debug("Writing XML to '{0}'".format(output)) output = utils.wrap_stream(output) try: output.write(job.output()) except IOError as exc: if exc.errno == errno.EPIPE: # EPIPE could happen if piping output to something # that doesn't read the whole input (e.g.: the UNIX # `head` command) return raise continue output_fn = os.path.join(output, job.name) logger.debug("Writing XML to '{0}'".format(output_fn)) with io.open(output_fn, 'w', encoding='utf-8') as f: f.write(job.output().decode('utf-8')) return self.parser.xml_jobs, len(self.parser.xml_jobs) # Filter out the jobs that did not change logging.debug('Filtering %d jobs for changed jobs', len(self.parser.xml_jobs)) step = time.time() jobs = [job for job in self.parser.xml_jobs if self.changed(job)] logging.debug("Filtered for changed jobs in %ss", (time.time() - step)) if not jobs: return [], 0 # Update the jobs logging.debug('Updating jobs') step = time.time() p_params = [{'job': job} for job in jobs] results = self.parallel_update_job( n_workers=n_workers, parallelize=p_params) logging.debug("Parsing results") # generalize the result parsing, as a parallelized job always returns a # list if len(p_params) in (1, 0): results = [results] for result in results: if isinstance(result, Exception): raise result else: # update in-memory cache j_name, j_md5 = result self.cache.set(j_name, j_md5) # write cache to disk self.cache.save() logging.debug("Updated %d jobs in %ss", len(jobs), time.time() - step) logging.debug("Total run took %ss", (time.time() - orig)) return jobs, len(jobs) @parallelize def parallel_update_job(self, job): self.jenkins.update_job(job.name, job.output().decode('utf-8')) return (job.name, job.md5()) def update_job(self, input_fn, jobs_glob=None, output=None): logging.warn('Current update_job function signature is deprecated and ' 'will change in future versions to the signature of the ' 'new parallel_update_job') return self.update_jobs(input_fn, jobs_glob, output)
class Builder(object): def __init__(self, jenkins_url, jenkins_user, jenkins_password, config=None, ignore_cache=False, flush_cache=False, plugins_list=None): self.jenkins = Jenkins(jenkins_url, jenkins_user, jenkins_password) self.cache = CacheStorage(jenkins_url, flush=flush_cache) self.global_config = config self.ignore_cache = ignore_cache self._plugins_list = plugins_list @property def plugins_list(self): if self._plugins_list is None: self._plugins_list = self.jenkins.get_plugins_info() return self._plugins_list def load_files(self, fn): self.parser = YamlParser(self.global_config, self.plugins_list) # handle deprecated behavior if not hasattr(fn, '__iter__'): logger.warning( 'Passing single elements for the `fn` argument in ' 'Builder.load_files is deprecated. Please update your code ' 'to use a list as support for automatic conversion will be ' 'removed in a future version.') fn = [fn] files_to_process = [] for path in fn: if os.path.isdir(path): files_to_process.extend([ os.path.join(path, f) for f in os.listdir(path) if (f.endswith('.yml') or f.endswith('.yaml')) ]) else: files_to_process.append(path) # symlinks used to allow loading of sub-dirs can result in duplicate # definitions of macros and templates when loading all from top-level unique_files = [] for f in files_to_process: rpf = os.path.realpath(f) if rpf not in unique_files: unique_files.append(rpf) else: logger.warning("File '%s' already added as '%s', ignoring " "reference to avoid duplicating yaml " "definitions." % (f, rpf)) for in_file in unique_files: # use of ask-for-permissions instead of ask-for-forgiveness # performs better when low use cases. if hasattr(in_file, 'name'): fname = in_file.name else: fname = in_file logger.debug("Parsing YAML file {0}".format(fname)) if hasattr(in_file, 'read'): self.parser.parse_fp(in_file) else: self.parser.parse(in_file) def delete_old_managed(self, keep): jobs = self.jenkins.get_jobs() deleted_jobs = 0 for job in jobs: if job['name'] not in keep and \ self.jenkins.is_managed(job['name']): logger.info("Removing obsolete jenkins job {0}".format( job['name'])) self.delete_job(job['name']) deleted_jobs += 1 else: logger.debug("Ignoring unmanaged jenkins job %s", job['name']) return deleted_jobs def delete_job(self, jobs_glob, fn=None): if fn: self.load_files(fn) self.parser.expandYaml([jobs_glob]) jobs = [j['name'] for j in self.parser.jobs] else: jobs = [jobs_glob] if jobs is not None: logger.info("Removing jenkins job(s): %s" % ", ".join(jobs)) for job in jobs: self.jenkins.delete_job(job) if (self.cache.is_cached(job)): self.cache.set(job, '') def delete_all_jobs(self): jobs = self.jenkins.get_jobs() logger.info("Number of jobs to delete: %d", len(jobs)) for job in jobs: self.delete_job(job['name']) def update_job(self, input_fn, jobs_glob=None, output=None): self.load_files(input_fn) self.parser.expandYaml(jobs_glob) self.parser.generateXML() logger.info("Number of jobs generated: %d", len(self.parser.xml_jobs)) self.parser.xml_jobs.sort(key=operator.attrgetter('name')) if (output and not hasattr(output, 'write') and not os.path.isdir(output)): logger.info("Creating directory %s" % output) try: os.makedirs(output) except OSError: if not os.path.isdir(output): raise updated_jobs = 0 for job in self.parser.xml_jobs: if output: if hasattr(output, 'write'): # `output` is a file-like object logger.info("Job name: %s", job.name) logger.debug("Writing XML to '{0}'".format(output)) try: output.write(job.output()) except IOError as exc: if exc.errno == errno.EPIPE: # EPIPE could happen if piping output to something # that doesn't read the whole input (e.g.: the UNIX # `head` command) return raise continue output_fn = os.path.join(output, job.name) logger.debug("Writing XML to '{0}'".format(output_fn)) f = open(output_fn, 'w') f.write(job.output()) f.close() continue md5 = job.md5() if (self.jenkins.is_job(job.name) and not self.cache.is_cached(job.name)): old_md5 = self.jenkins.get_job_md5(job.name) self.cache.set(job.name, old_md5) if self.cache.has_changed(job.name, md5) or self.ignore_cache: self.jenkins.update_job(job.name, job.output()) updated_jobs += 1 self.cache.set(job.name, md5) else: logger.debug("'{0}' has not changed".format(job.name)) return self.parser.xml_jobs, updated_jobs