def set_input_state(self, job, build_nr, state): """Set the state of input :param job: :param build_nr: :param state: :return: """ try: for build in self._specification["input"]["builds"][job]: if build["build"] == build_nr: build["status"] = state break else: raise PresentationError("Build '{}' is not defined for job '{}'" " in specification file.". format(build_nr, job)) except KeyError: raise PresentationError("Job '{}' and build '{}' is not defined in " "specification file.".format(job, build_nr))
def _replace_tags(self, data, src_data=None): """Replace tag(s) in the data by their values. :param data: The data where the tags will be replaced by their values. :param src_data: Data where the tags are defined. It is dictionary where the key is the tag and the value is the tag value. If not given, 'data' is used instead. :type data: str or dict :type src_data: dict :returns: Data with the tags replaced. :rtype: str or dict :raises: PresentationError if it is not possible to replace the tag or the data is not the supported data type (str, dict). """ if src_data is None: src_data = data if isinstance(data, str): tag = self._find_tag(data) if tag is not None: data = data.replace(tag, src_data[tag[1:-1]]) elif isinstance(data, dict): counter = 0 for key, value in data.items(): tag = self._find_tag(value) if tag is not None: try: data[key] = value.replace(tag, src_data[tag[1:-1]]) counter += 1 except KeyError: raise PresentationError("Not possible to replace the " "tag '{}'".format(tag)) if counter: self._replace_tags(data, src_data) else: raise PresentationError("Replace tags: Not supported data type.") return data
def _parse_input(self): """Parse input specification in the specification YAML file. :raises: PresentationError if there are no data to process. """ logging.info("Parsing specification file: input ...") idx = self._get_type_index("input") if idx is None: raise PresentationError("No data to process.") try: for key, value in self._cfg_yaml[idx]["general"].items(): self._specification["input"][key] = value self._specification["input"]["builds"] = dict() for job, builds in self._cfg_yaml[idx]["builds"].items(): if builds: if isinstance(builds, dict): build_nr = builds.get("end", None) try: build_nr = int(build_nr) except ValueError: # defined as a range <start, build_type> build_nr = self._get_build_number(job, build_nr) builds = [x for x in range(builds["start"], build_nr+1)] self._specification["input"]["builds"][job] = list() for build in builds: self._specification["input"]["builds"][job]. \ append({"build": build, "status": None}) else: logging.warning("No build is defined for the job '{}'. " "Trying to continue without it.". format(job)) except KeyError: raise PresentationError("No data to process.") logging.info("Done.")
def _make_dirs(self): """Create the directories specified in the 'make-dirs' part of 'environment' section in the specification file. :raises: PresentationError if it is not possible to remove or create a directory. """ if self._force: logging.info("Removing old build(s) ...") for directory in self._env["build-dirs"]: dir_to_remove = self._env["paths"][directory] if os.path.isdir(dir_to_remove): try: shutil.rmtree(dir_to_remove) logging.info(" Removed: {}".format(dir_to_remove)) except OSError: raise PresentationError("Cannot remove the directory " "'{}'".format(dir_to_remove)) logging.info("Done.") logging.info("Making directories ...") for directory in self._env["make-dirs"]: dir_to_make = self._env["paths"][directory] try: if os.path.isdir(dir_to_make): logging.warning( "The directory '{}' exists, skipping.".format( dir_to_make)) else: os.makedirs(dir_to_make) logging.info(" Created: {}".format(dir_to_make)) except OSError: raise PresentationError( "Cannot make the directory '{}'".format(dir_to_make)) logging.info("Done.")
def _get_build_number(self, job, build_type): """Get the number of the job defined by its name: - lastSuccessfulBuild - lastCompletedBuild :param job: Job name. :param build_type: Build type: - lastSuccessfulBuild - lastCompletedBuild :type job" str :raises PresentationError: If it is not possible to get the build number. :returns: The build number. :rtype: int """ # defined as a range <start, end> if build_type == "lastSuccessfulBuild": # defined as a range <start, lastSuccessfulBuild> ret_code, build_nr, _ = get_last_successful_build_number( self.environment["urls"]["URL[JENKINS,CSIT]"], job) elif build_type == "lastCompletedBuild": # defined as a range <start, lastCompletedBuild> ret_code, build_nr, _ = get_last_completed_build_number( self.environment["urls"]["URL[JENKINS,CSIT]"], job) else: raise PresentationError( "Not supported build type: '{0}'".format(build_type)) if ret_code != 0: raise PresentationError("Not possible to get the number of the " "build number.") try: build_nr = int(build_nr) return build_nr except ValueError as err: raise PresentationError("Not possible to get the number of the " "build number.\nReason: {0}".format(err))
def _read_csv_template(file_name): """Read the template from a .csv file. :param file_name: Name / full path / relative path of the file to read. :type file_name: str :returns: Data from the template as list (lines) of lists (items on line). :rtype: list :raises: PresentationError if it is not possible to read the file. """ try: with open(file_name, 'r') as csv_file: tmpl_data = list() for line in csv_file: tmpl_data.append(line[:-1].split(",")) return tmpl_data except IOError as err: raise PresentationError(str(err), level="ERROR")
def _parse_configuration(self): """Parse configuration of PAL in the specification YAML file. """ logging.info("Parsing specification file: configuration ...") idx = self._get_type_index("configuration") if idx is None: logging.warning( "No configuration information in the specification " "file.") return None try: self._specification["configuration"] = self._cfg_yaml[idx] except KeyError: raise PresentationError("No configuration defined.") logging.info("Done.")
def read_specification(self): """Parse specification in the specification YAML file. :raises: PresentationError if an error occurred while parsing the specification file. """ try: self._cfg_yaml = load(self._cfg_file) except YAMLError as err: raise PresentationError(msg="An error occurred while parsing the " "specification file.", details=str(err)) self._parse_env() self._parse_configuration() self._parse_input() self._parse_output() self._parse_static() self._parse_elements() logging.debug("Specification: \n{}".format(pformat( self._specification)))
def _parse_debug(self): """Parse debug specification in the specification YAML file. """ if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1: return None logging.info("Parsing specification file: debug ...") idx = self._get_type_index("debug") if idx is None: self.environment["configuration"]["CFG[DEBUG]"] = 0 return None try: for key, value in self._cfg_yaml[idx]["general"].items(): self._specification["debug"][key] = value self._specification["input"]["builds"] = dict() for job, builds in self._cfg_yaml[idx]["builds"].items(): if builds: self._specification["input"]["builds"][job] = list() for build in builds: self._specification["input"]["builds"][job].\ append({"build": build["build"], "status": "downloaded", "file-name": self._replace_tags( build["file"], self.environment["paths"])}) else: logging.warning( "No build is defined for the job '{}'. " "Trying to continue without it.".format(job)) except KeyError: raise PresentationError("No data to process.")
def download_and_unzip_data_file(spec, job, build, pid, log): """Download and unzip a source file. :param spec: Specification read form the specification file. :param job: Name of the Jenkins job. :param build: Information about the build. :param pid: PID of the process executing this method. :param log: List of log messages. :type spec: Specification :type job: str :type build: dict :type pid: int :type log: list of tuples (severity, msg) :returns: True if the download was successful, otherwise False. :rtype: bool """ if job.startswith("csit-"): if spec.input["file-name"].endswith(".zip"): url = spec.environment["urls"]["URL[JENKINS,CSIT]"] elif spec.input["file-name"].endswith(".gz"): url = spec.environment["urls"]["URL[NEXUS,LOG]"] else: log.append(("ERROR", "Not supported file format.")) return False elif job.startswith("hc2vpp-"): url = spec.environment["urls"]["URL[JENKINS,HC]"] else: raise PresentationError("No url defined for the job '{}'.".format(job)) file_name = spec.input["file-name"] full_name = spec.input["download-path"]. \ format(job=job, build=build["build"], filename=file_name) url = "{0}/{1}".format(url, full_name) new_name = join( spec.environment["paths"]["DIR[WORKING,DATA]"], "{job}{sep}{build}{sep}{name}".format(job=job, sep=SEPARATOR, build=build["build"], name=file_name)) # Download the file from the defined source (Jenkins, logs.fd.io): success = _download_file(url, new_name, log) if success and new_name.endswith(".zip"): if not is_zipfile(new_name): success = False # If not successful, download from docs.fd.io: if not success: log.append(("INFO", " Trying to download from https://docs.fd.io:")) release = re.search(REGEX_RELEASE, job).group(2) for rls in (release, "master"): nexus_file_name = "{job}{sep}{build}{sep}{name}". \ format(job=job, sep=SEPARATOR, build=build["build"], name=file_name) try: rls = "rls{0}".format(int(rls)) except ValueError: pass url = "{url}/{release}/{dir}/{file}". \ format(url=spec.environment["urls"]["URL[NEXUS]"], release=rls, dir=spec.environment["urls"]["DIR[NEXUS]"], file=nexus_file_name) success = _download_file(url, new_name, log) if success: break if success: build["file-name"] = new_name else: return False if spec.input["file-name"].endswith(".gz"): if "docs.fd.io" in url: execute_command( "gzip --decompress --keep --force {0}".format(new_name)) else: rename(new_name, new_name[:-3]) execute_command("gzip --keep {0}".format(new_name[:-3])) build["file-name"] = new_name[:-3] if new_name.endswith(".zip"): if is_zipfile(new_name): return _unzip_file(spec, build, pid, log) else: log.append( ("ERROR", "Zip file '{0}' is corrupted.".format(new_name))) return False else: return True
def download_data_files(spec): """Download all data specified in the specification file in the section type: input --> builds. :param spec: Specification. :type spec: Specification :raises: PresentationError if there is no url defined for the job. """ for job, builds in spec.builds.items(): for build in builds: if job.startswith("csit-"): url = spec.environment["urls"]["URL[JENKINS,CSIT]"] elif job.startswith("hc2vpp-"): url = spec.environment["urls"]["URL[JENKINS,HC]"] else: raise PresentationError( "No url defined for the job '{}'.".format(job)) file_name = spec.input["file-name"] full_name = spec.input["download-path"].\ format(job=job, build=build["build"], filename=file_name) url = "{0}/{1}".format(url, full_name) new_name = join( spec.environment["paths"]["DIR[WORKING,DATA]"], "{job}{sep}{build}{sep}{name}".format(job=job, sep=SEPARATOR, build=build["build"], name=file_name)) logging.info("Downloading the file '{0}' to '{1}'.".format( url, new_name)) status = "failed" try: response = get(url, stream=True) code = response.status_code if code != codes["OK"]: logging.error("{0}: {1}".format(code, responses[code])) spec.set_input_state(job, build["build"], "not found") continue file_handle = open(new_name, "wb") for chunk in response.iter_content(chunk_size=CHUNK_SIZE): if chunk: file_handle.write(chunk) file_handle.close() expected_length = None try: expected_length = int(response.headers["Content-Length"]) logging.debug( " Expected file size: {0}B".format(expected_length)) except KeyError: logging.debug(" No information about expected size.") real_length = getsize(new_name) logging.debug(" Downloaded size: {0}B".format(real_length)) if expected_length: if real_length == expected_length: status = "downloaded" logging.info("{0}: {1}".format(code, responses[code])) else: logging.error( "The file size differs from the expected " "size.") else: status = "downloaded" logging.info("{0}: {1}".format(code, responses[code])) except ConnectionError as err: logging.error("Not possible to connect to '{0}'.".format(url)) logging.debug(err) except HTTPError as err: logging.error("Invalid HTTP response from '{0}'.".format(url)) logging.debug(err) except TooManyRedirects as err: logging.error("Request exceeded the configured number " "of maximum re-directions.") logging.debug(err) except Timeout as err: logging.error("Request timed out.") logging.debug(err) except RequestException as err: logging.error("Unexpected HTTP request exception.") logging.debug(err) except (IOError, ValueError, KeyError) as err: logging.error("Download failed.") logging.debug("Reason: {0}".format(err)) spec.set_input_state(job, build["build"], status) spec.set_input_file_name(job, build["build"], new_name) if status == "failed": logging.info("Removing the file '{0}'".format(new_name)) try: remove(new_name) except OSError as err: logging.warning(str(err)) spec.set_input_file_name(job, build["build"], None) unzip_files(spec)
def unzip_files(spec): """Unzip downloaded zip files :param spec: Specification. :type spec: Specification :raises: PresentationError if the zip file does not exist or it is not a zip file. """ if spec.is_debug: data_file = spec.debug["extract"] else: data_file = spec.input["extract"] for job, builds in spec.builds.items(): for build in builds: if build["status"] == "failed" or build["status"] == "not found": continue try: status = "failed" directory = spec.environment["paths"]["DIR[WORKING,DATA]"] file_name = join(build["file-name"]) if build["status"] == "downloaded" and is_zipfile(file_name): logging.info("Unziping: '{0}' from '{1}'.".format( data_file, file_name)) new_name = "{0}{1}{2}".format( file_name.rsplit('.')[-2], SEPARATOR, data_file.split("/")[-1]) try: with ZipFile(file_name, 'r') as zip_file: zip_file.extract(data_file, directory) logging.info("Moving {0} to {1} ...".format( join(directory, data_file), directory)) move(join(directory, data_file), directory) logging.info("Renaming the file '{0}' to '{1}'".format( join(directory, data_file.split("/")[-1]), new_name)) rename(join(directory, data_file.split("/")[-1]), new_name) status = "unzipped" spec.set_input_state(job, build["build"], status) spec.set_input_file_name(job, build["build"], new_name) except (BadZipfile, RuntimeError) as err: logging.error( "Failed to unzip the file '{0}': {1}.".format( file_name, str(err))) except OSError as err: logging.error( "Failed to rename the file '{0}': {1}.".format( data_file, str(err))) finally: if status == "failed": spec.set_input_file_name(job, build["build"], None) else: raise PresentationError( "The file '{0}' does not exist or " "it is not a zip file".format(file_name)) spec.set_input_state(job, build["build"], status) except KeyError: pass
def _parse_elements(self): """Parse elements (tables, plots) specification in the specification YAML file. """ logging.info("Parsing specification file: elements ...") count = 1 for element in self._cfg_yaml: try: element["output-file"] = self._replace_tags( element["output-file"], self._specification["environment"]["paths"]) except KeyError: pass try: element["input-file"] = self._replace_tags( element["input-file"], self._specification["environment"]["paths"]) except KeyError: pass # add data sets to the elements: if isinstance(element.get("data", None), str): data_set = element["data"] try: element["data"] = self.configuration["data-sets"][data_set] except KeyError: raise PresentationError( "Data set {0} is not defined in " "the configuration section.".format(data_set)) if element["type"] == "table": logging.info(" {:3d} Processing a table ...".format(count)) try: element["template"] = self._replace_tags( element["template"], self._specification["environment"]["paths"]) except KeyError: pass # add data sets try: for item in ("reference", "compare"): if element.get(item, None): data_set = element[item].get("data", None) if isinstance(data_set, str): element[item]["data"] = \ self.configuration["data-sets"][data_set] if element.get("history", None): for i in range(len(element["history"])): data_set = element["history"][i].get("data", None) if isinstance(data_set, str): element["history"][i]["data"] = \ self.configuration["data-sets"][data_set] except KeyError: raise PresentationError( "Wrong data set used in {0}.".format( element.get("title", ""))) self._specification["tables"].append(element) count += 1 elif element["type"] == "plot": logging.info(" {:3d} Processing a plot ...".format(count)) # Add layout to the plots: layout = element["layout"].get("layout", None) if layout is not None: element["layout"].pop("layout") try: for key, val in (self.configuration["plot-layouts"] [layout].items()): element["layout"][key] = val except KeyError: raise PresentationError( "Layout {0} is not defined in " "the configuration section.".format(layout)) self._specification["plots"].append(element) count += 1 elif element["type"] == "file": logging.info(" {:3d} Processing a file ...".format(count)) try: element["dir-tables"] = self._replace_tags( element["dir-tables"], self._specification["environment"]["paths"]) except KeyError: pass self._specification["files"].append(element) count += 1 elif element["type"] == "cpta": logging.info(" {:3d} Processing Continuous Performance " "Trending and Analysis ...".format(count)) for plot in element["plots"]: # Add layout to the plots: layout = plot.get("layout", None) if layout is not None: try: plot["layout"] = \ self.configuration["plot-layouts"][layout] except KeyError: raise PresentationError( "Layout {0} is not defined in the " "configuration section.".format(layout)) # Add data sets: if isinstance(plot.get("data", None), str): data_set = plot["data"] try: plot["data"] = \ self.configuration["data-sets"][data_set] except KeyError: raise PresentationError( "Data set {0} is not defined in " "the configuration section.".format(data_set)) self._specification["cpta"] = element count += 1 logging.info("Done.")
def _parse_configuration(self): """Parse configuration of PAL in the specification YAML file. """ logging.info("Parsing specification file: configuration ...") idx = self._get_type_index("configuration") if idx is None: logging.warning( "No configuration information in the specification " "file.") return None try: self._specification["configuration"] = self._cfg_yaml[idx] except KeyError: raise PresentationError("No configuration defined.") # Data sets: Replace ranges by lists for set_name, data_set in self.configuration["data-sets"].items(): if not isinstance(data_set, dict): continue for job, builds in data_set.items(): if builds: if isinstance(builds, dict): build_end = builds.get("end", None) try: build_end = int(build_end) except ValueError: # defined as a range <start, build_type> build_end = self._get_build_number(job, build_end) builds = [ x for x in range(builds["start"], build_end + 1) if x not in builds.get("skip", list()) ] self.configuration["data-sets"][set_name][job] = builds elif isinstance(builds, list): for idx, item in enumerate(builds): try: builds[idx] = int(item) except ValueError: # defined as a range <build_type> builds[idx] = self._get_build_number(job, item) # Data sets: add sub-sets to sets (only one level): for set_name, data_set in self.configuration["data-sets"].items(): if isinstance(data_set, list): new_set = dict() for item in data_set: try: for key, val in self.configuration["data-sets"][item].\ items(): new_set[key] = val except KeyError: raise PresentationError( "Data set {0} is not defined in " "the configuration section.".format(item)) self.configuration["data-sets"][set_name] = new_set # Mapping table: mapping = None mapping_file_name = self._specification["configuration"].\ get("mapping-file", None) if mapping_file_name: logging.debug("Mapping file: '{0}'".format(mapping_file_name)) try: with open(mapping_file_name, 'r') as mfile: mapping = load(mfile) logging.debug("Loaded mapping table:\n{0}".format(mapping)) except (YAMLError, IOError) as err: raise PresentationError( msg="An error occurred while parsing the mapping file " "'{0}'.".format(mapping_file_name), details=repr(err)) # Make sure everything is lowercase if mapping: self._specification["configuration"]["mapping"] = \ {key.lower(): val.lower() for key, val in mapping.iteritems()} else: self._specification["configuration"]["mapping"] = dict() # Ignore list: ignore = None ignore_list_name = self._specification["configuration"].\ get("ignore-list", None) if ignore_list_name: logging.debug("Ignore list file: '{0}'".format(ignore_list_name)) try: with open(ignore_list_name, 'r') as ifile: ignore = load(ifile) logging.debug("Loaded ignore list:\n{0}".format(ignore)) except (YAMLError, IOError) as err: raise PresentationError( msg="An error occurred while parsing the ignore list file " "'{0}'.".format(ignore_list_name), details=repr(err)) # Make sure everything is lowercase if ignore: self._specification["configuration"]["ignore"] = \ [item.lower() for item in ignore] else: self._specification["configuration"]["ignore"] = list() logging.info("Done.")
def _parse_elements(self): """Parse elements (tables, plots) specification in the specification YAML file. """ logging.info("Parsing specification file: elements ...") count = 1 for element in self._cfg_yaml: try: element["output-file"] = self._replace_tags( element["output-file"], self._specification["environment"]["paths"]) except KeyError: pass # add data sets to the elements: if isinstance(element.get("data", None), str): data_set = element["data"] try: element["data"] = self.configuration["data-sets"][data_set] except KeyError: raise PresentationError( "Data set {0} is not defined in " "the configuration section.".format(data_set)) if element["type"] == "table": logging.info(" {:3d} Processing a table ...".format(count)) try: element["template"] = self._replace_tags( element["template"], self._specification["environment"]["paths"]) except KeyError: pass self._specification["tables"].append(element) count += 1 elif element["type"] == "plot": logging.info(" {:3d} Processing a plot ...".format(count)) # Add layout to the plots: layout = element["layout"].get("layout", None) if layout is not None: element["layout"].pop("layout") try: for key, val in (self.configuration["plot-layouts"] [layout].items()): element["layout"][key] = val except KeyError: raise PresentationError( "Layout {0} is not defined in " "the configuration section.".format(layout)) self._specification["plots"].append(element) count += 1 elif element["type"] == "file": logging.info(" {:3d} Processing a file ...".format(count)) try: element["dir-tables"] = self._replace_tags( element["dir-tables"], self._specification["environment"]["paths"]) except KeyError: pass self._specification["files"].append(element) count += 1 logging.info("Done.")