def set_input_state(self, job, build_nr, state):
"""Set the state of input
:param job:
:param build_nr:
:param state:
:return:
"""
try:
for build in self._specification["input"]["builds"][job]:
if build["build"] == build_nr:
build["status"] = state
break
else:
raise PresentationError("Build '{}' is not defined for job '{}'"
" in specification file.".
format(build_nr, job))
except KeyError:
raise PresentationError("Job '{}' and build '{}' is not defined in "
"specification file.".format(job, build_nr))
def _replace_tags(self, data, src_data=None):
"""Replace tag(s) in the data by their values.
:param data: The data where the tags will be replaced by their values.
:param src_data: Data where the tags are defined. It is dictionary where
the key is the tag and the value is the tag value. If not given, 'data'
is used instead.
:type data: str or dict
:type src_data: dict
:returns: Data with the tags replaced.
:rtype: str or dict
:raises: PresentationError if it is not possible to replace the tag or
the data is not the supported data type (str, dict).
"""
if src_data is None:
src_data = data
if isinstance(data, str):
tag = self._find_tag(data)
if tag is not None:
data = data.replace(tag, src_data[tag[1:-1]])
elif isinstance(data, dict):
counter = 0
for key, value in data.items():
tag = self._find_tag(value)
if tag is not None:
try:
data[key] = value.replace(tag, src_data[tag[1:-1]])
counter += 1
except KeyError:
raise PresentationError("Not possible to replace the "
"tag '{}'".format(tag))
if counter:
self._replace_tags(data, src_data)
else:
raise PresentationError("Replace tags: Not supported data type.")
return data
def _parse_input(self):
"""Parse input specification in the specification YAML file.
:raises: PresentationError if there are no data to process.
"""
logging.info("Parsing specification file: input ...")
idx = self._get_type_index("input")
if idx is None:
raise PresentationError("No data to process.")
try:
for key, value in self._cfg_yaml[idx]["general"].items():
self._specification["input"][key] = value
self._specification["input"]["builds"] = dict()
for job, builds in self._cfg_yaml[idx]["builds"].items():
if builds:
if isinstance(builds, dict):
build_nr = builds.get("end", None)
try:
build_nr = int(build_nr)
except ValueError:
# defined as a range <start, build_type>
build_nr = self._get_build_number(job, build_nr)
builds = [x for x in range(builds["start"], build_nr+1)]
self._specification["input"]["builds"][job] = list()
for build in builds:
self._specification["input"]["builds"][job]. \
append({"build": build, "status": None})
else:
logging.warning("No build is defined for the job '{}'. "
"Trying to continue without it.".
format(job))
except KeyError:
raise PresentationError("No data to process.")
logging.info("Done.")
def _make_dirs(self):
"""Create the directories specified in the 'make-dirs' part of
'environment' section in the specification file.
:raises: PresentationError if it is not possible to remove or create a
directory.
"""
if self._force:
logging.info("Removing old build(s) ...")
for directory in self._env["build-dirs"]:
dir_to_remove = self._env["paths"][directory]
if os.path.isdir(dir_to_remove):
try:
shutil.rmtree(dir_to_remove)
logging.info(" Removed: {}".format(dir_to_remove))
except OSError:
raise PresentationError("Cannot remove the directory "
"'{}'".format(dir_to_remove))
logging.info("Done.")
logging.info("Making directories ...")
for directory in self._env["make-dirs"]:
dir_to_make = self._env["paths"][directory]
try:
if os.path.isdir(dir_to_make):
logging.warning(
"The directory '{}' exists, skipping.".format(
dir_to_make))
else:
os.makedirs(dir_to_make)
logging.info(" Created: {}".format(dir_to_make))
except OSError:
raise PresentationError(
"Cannot make the directory '{}'".format(dir_to_make))
logging.info("Done.")
def _get_build_number(self, job, build_type):
"""Get the number of the job defined by its name:
- lastSuccessfulBuild
- lastCompletedBuild
:param job: Job name.
:param build_type: Build type:
- lastSuccessfulBuild
- lastCompletedBuild
:type job" str
:raises PresentationError: If it is not possible to get the build
number.
:returns: The build number.
:rtype: int
"""
# defined as a range <start, end>
if build_type == "lastSuccessfulBuild":
# defined as a range <start, lastSuccessfulBuild>
ret_code, build_nr, _ = get_last_successful_build_number(
self.environment["urls"]["URL[JENKINS,CSIT]"], job)
elif build_type == "lastCompletedBuild":
# defined as a range <start, lastCompletedBuild>
ret_code, build_nr, _ = get_last_completed_build_number(
self.environment["urls"]["URL[JENKINS,CSIT]"], job)
else:
raise PresentationError(
"Not supported build type: '{0}'".format(build_type))
if ret_code != 0:
raise PresentationError("Not possible to get the number of the "
"build number.")
try:
build_nr = int(build_nr)
return build_nr
except ValueError as err:
raise PresentationError("Not possible to get the number of the "
"build number.\nReason: {0}".format(err))
def _read_csv_template(file_name):
"""Read the template from a .csv file.
:param file_name: Name / full path / relative path of the file to read.
:type file_name: str
:returns: Data from the template as list (lines) of lists (items on line).
:rtype: list
:raises: PresentationError if it is not possible to read the file.
"""
try:
with open(file_name, 'r') as csv_file:
tmpl_data = list()
for line in csv_file:
tmpl_data.append(line[:-1].split(","))
return tmpl_data
except IOError as err:
raise PresentationError(str(err), level="ERROR")
def _parse_configuration(self):
"""Parse configuration of PAL in the specification YAML file.
"""
logging.info("Parsing specification file: configuration ...")
idx = self._get_type_index("configuration")
if idx is None:
logging.warning(
"No configuration information in the specification "
"file.")
return None
try:
self._specification["configuration"] = self._cfg_yaml[idx]
except KeyError:
raise PresentationError("No configuration defined.")
logging.info("Done.")
def read_specification(self):
"""Parse specification in the specification YAML file.
:raises: PresentationError if an error occurred while parsing the
specification file.
"""
try:
self._cfg_yaml = load(self._cfg_file)
except YAMLError as err:
raise PresentationError(msg="An error occurred while parsing the "
"specification file.",
details=str(err))
self._parse_env()
self._parse_configuration()
self._parse_input()
self._parse_output()
self._parse_static()
self._parse_elements()
logging.debug("Specification: \n{}".format(pformat(
self._specification)))
def _parse_debug(self):
"""Parse debug specification in the specification YAML file.
"""
if int(self.environment["configuration"]["CFG[DEBUG]"]) != 1:
return None
logging.info("Parsing specification file: debug ...")
idx = self._get_type_index("debug")
if idx is None:
self.environment["configuration"]["CFG[DEBUG]"] = 0
return None
try:
for key, value in self._cfg_yaml[idx]["general"].items():
self._specification["debug"][key] = value
self._specification["input"]["builds"] = dict()
for job, builds in self._cfg_yaml[idx]["builds"].items():
if builds:
self._specification["input"]["builds"][job] = list()
for build in builds:
self._specification["input"]["builds"][job].\
append({"build": build["build"],
"status": "downloaded",
"file-name": self._replace_tags(
build["file"],
self.environment["paths"])})
else:
logging.warning(
"No build is defined for the job '{}'. "
"Trying to continue without it.".format(job))
except KeyError:
raise PresentationError("No data to process.")
def download_and_unzip_data_file(spec, job, build, pid, log):
"""Download and unzip a source file.
:param spec: Specification read form the specification file.
:param job: Name of the Jenkins job.
:param build: Information about the build.
:param pid: PID of the process executing this method.
:param log: List of log messages.
:type spec: Specification
:type job: str
:type build: dict
:type pid: int
:type log: list of tuples (severity, msg)
:returns: True if the download was successful, otherwise False.
:rtype: bool
"""
if job.startswith("csit-"):
if spec.input["file-name"].endswith(".zip"):
url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
elif spec.input["file-name"].endswith(".gz"):
url = spec.environment["urls"]["URL[NEXUS,LOG]"]
else:
log.append(("ERROR", "Not supported file format."))
return False
elif job.startswith("hc2vpp-"):
url = spec.environment["urls"]["URL[JENKINS,HC]"]
else:
raise PresentationError("No url defined for the job '{}'.".format(job))
file_name = spec.input["file-name"]
full_name = spec.input["download-path"]. \
format(job=job, build=build["build"], filename=file_name)
url = "{0}/{1}".format(url, full_name)
new_name = join(
spec.environment["paths"]["DIR[WORKING,DATA]"],
"{job}{sep}{build}{sep}{name}".format(job=job,
sep=SEPARATOR,
build=build["build"],
name=file_name))
# Download the file from the defined source (Jenkins, logs.fd.io):
success = _download_file(url, new_name, log)
if success and new_name.endswith(".zip"):
if not is_zipfile(new_name):
success = False
# If not successful, download from docs.fd.io:
if not success:
log.append(("INFO", " Trying to download from https://docs.fd.io:"))
release = re.search(REGEX_RELEASE, job).group(2)
for rls in (release, "master"):
nexus_file_name = "{job}{sep}{build}{sep}{name}". \
format(job=job, sep=SEPARATOR, build=build["build"],
name=file_name)
try:
rls = "rls{0}".format(int(rls))
except ValueError:
pass
url = "{url}/{release}/{dir}/{file}". \
format(url=spec.environment["urls"]["URL[NEXUS]"],
release=rls,
dir=spec.environment["urls"]["DIR[NEXUS]"],
file=nexus_file_name)
success = _download_file(url, new_name, log)
if success:
break
if success:
build["file-name"] = new_name
else:
return False
if spec.input["file-name"].endswith(".gz"):
if "docs.fd.io" in url:
execute_command(
"gzip --decompress --keep --force {0}".format(new_name))
else:
rename(new_name, new_name[:-3])
execute_command("gzip --keep {0}".format(new_name[:-3]))
build["file-name"] = new_name[:-3]
if new_name.endswith(".zip"):
if is_zipfile(new_name):
return _unzip_file(spec, build, pid, log)
else:
log.append(
("ERROR", "Zip file '{0}' is corrupted.".format(new_name)))
return False
else:
return True
def download_data_files(spec):
"""Download all data specified in the specification file in the section
type: input --> builds.
:param spec: Specification.
:type spec: Specification
:raises: PresentationError if there is no url defined for the job.
"""
for job, builds in spec.builds.items():
for build in builds:
if job.startswith("csit-"):
url = spec.environment["urls"]["URL[JENKINS,CSIT]"]
elif job.startswith("hc2vpp-"):
url = spec.environment["urls"]["URL[JENKINS,HC]"]
else:
raise PresentationError(
"No url defined for the job '{}'.".format(job))
file_name = spec.input["file-name"]
full_name = spec.input["download-path"].\
format(job=job, build=build["build"], filename=file_name)
url = "{0}/{1}".format(url, full_name)
new_name = join(
spec.environment["paths"]["DIR[WORKING,DATA]"],
"{job}{sep}{build}{sep}{name}".format(job=job,
sep=SEPARATOR,
build=build["build"],
name=file_name))
logging.info("Downloading the file '{0}' to '{1}'.".format(
url, new_name))
status = "failed"
try:
response = get(url, stream=True)
code = response.status_code
if code != codes["OK"]:
logging.error("{0}: {1}".format(code, responses[code]))
spec.set_input_state(job, build["build"], "not found")
continue
file_handle = open(new_name, "wb")
for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
if chunk:
file_handle.write(chunk)
file_handle.close()
expected_length = None
try:
expected_length = int(response.headers["Content-Length"])
logging.debug(
" Expected file size: {0}B".format(expected_length))
except KeyError:
logging.debug(" No information about expected size.")
real_length = getsize(new_name)
logging.debug(" Downloaded size: {0}B".format(real_length))
if expected_length:
if real_length == expected_length:
status = "downloaded"
logging.info("{0}: {1}".format(code, responses[code]))
else:
logging.error(
"The file size differs from the expected "
"size.")
else:
status = "downloaded"
logging.info("{0}: {1}".format(code, responses[code]))
except ConnectionError as err:
logging.error("Not possible to connect to '{0}'.".format(url))
logging.debug(err)
except HTTPError as err:
logging.error("Invalid HTTP response from '{0}'.".format(url))
logging.debug(err)
except TooManyRedirects as err:
logging.error("Request exceeded the configured number "
"of maximum re-directions.")
logging.debug(err)
except Timeout as err:
logging.error("Request timed out.")
logging.debug(err)
except RequestException as err:
logging.error("Unexpected HTTP request exception.")
logging.debug(err)
except (IOError, ValueError, KeyError) as err:
logging.error("Download failed.")
logging.debug("Reason: {0}".format(err))
spec.set_input_state(job, build["build"], status)
spec.set_input_file_name(job, build["build"], new_name)
if status == "failed":
logging.info("Removing the file '{0}'".format(new_name))
try:
remove(new_name)
except OSError as err:
logging.warning(str(err))
spec.set_input_file_name(job, build["build"], None)
unzip_files(spec)
def unzip_files(spec):
"""Unzip downloaded zip files
:param spec: Specification.
:type spec: Specification
:raises: PresentationError if the zip file does not exist or it is not a
zip file.
"""
if spec.is_debug:
data_file = spec.debug["extract"]
else:
data_file = spec.input["extract"]
for job, builds in spec.builds.items():
for build in builds:
if build["status"] == "failed" or build["status"] == "not found":
continue
try:
status = "failed"
directory = spec.environment["paths"]["DIR[WORKING,DATA]"]
file_name = join(build["file-name"])
if build["status"] == "downloaded" and is_zipfile(file_name):
logging.info("Unziping: '{0}' from '{1}'.".format(
data_file, file_name))
new_name = "{0}{1}{2}".format(
file_name.rsplit('.')[-2], SEPARATOR,
data_file.split("/")[-1])
try:
with ZipFile(file_name, 'r') as zip_file:
zip_file.extract(data_file, directory)
logging.info("Moving {0} to {1} ...".format(
join(directory, data_file), directory))
move(join(directory, data_file), directory)
logging.info("Renaming the file '{0}' to '{1}'".format(
join(directory,
data_file.split("/")[-1]), new_name))
rename(join(directory,
data_file.split("/")[-1]), new_name)
status = "unzipped"
spec.set_input_state(job, build["build"], status)
spec.set_input_file_name(job, build["build"], new_name)
except (BadZipfile, RuntimeError) as err:
logging.error(
"Failed to unzip the file '{0}': {1}.".format(
file_name, str(err)))
except OSError as err:
logging.error(
"Failed to rename the file '{0}': {1}.".format(
data_file, str(err)))
finally:
if status == "failed":
spec.set_input_file_name(job, build["build"], None)
else:
raise PresentationError(
"The file '{0}' does not exist or "
"it is not a zip file".format(file_name))
spec.set_input_state(job, build["build"], status)
except KeyError:
pass
def _parse_elements(self):
"""Parse elements (tables, plots) specification in the specification
YAML file.
"""
logging.info("Parsing specification file: elements ...")
count = 1
for element in self._cfg_yaml:
try:
element["output-file"] = self._replace_tags(
element["output-file"],
self._specification["environment"]["paths"])
except KeyError:
pass
try:
element["input-file"] = self._replace_tags(
element["input-file"],
self._specification["environment"]["paths"])
except KeyError:
pass
# add data sets to the elements:
if isinstance(element.get("data", None), str):
data_set = element["data"]
try:
element["data"] = self.configuration["data-sets"][data_set]
except KeyError:
raise PresentationError(
"Data set {0} is not defined in "
"the configuration section.".format(data_set))
if element["type"] == "table":
logging.info(" {:3d} Processing a table ...".format(count))
try:
element["template"] = self._replace_tags(
element["template"],
self._specification["environment"]["paths"])
except KeyError:
pass
# add data sets
try:
for item in ("reference", "compare"):
if element.get(item, None):
data_set = element[item].get("data", None)
if isinstance(data_set, str):
element[item]["data"] = \
self.configuration["data-sets"][data_set]
if element.get("history", None):
for i in range(len(element["history"])):
data_set = element["history"][i].get("data", None)
if isinstance(data_set, str):
element["history"][i]["data"] = \
self.configuration["data-sets"][data_set]
except KeyError:
raise PresentationError(
"Wrong data set used in {0}.".format(
element.get("title", "")))
self._specification["tables"].append(element)
count += 1
elif element["type"] == "plot":
logging.info(" {:3d} Processing a plot ...".format(count))
# Add layout to the plots:
layout = element["layout"].get("layout", None)
if layout is not None:
element["layout"].pop("layout")
try:
for key, val in (self.configuration["plot-layouts"]
[layout].items()):
element["layout"][key] = val
except KeyError:
raise PresentationError(
"Layout {0} is not defined in "
"the configuration section.".format(layout))
self._specification["plots"].append(element)
count += 1
elif element["type"] == "file":
logging.info(" {:3d} Processing a file ...".format(count))
try:
element["dir-tables"] = self._replace_tags(
element["dir-tables"],
self._specification["environment"]["paths"])
except KeyError:
pass
self._specification["files"].append(element)
count += 1
elif element["type"] == "cpta":
logging.info(" {:3d} Processing Continuous Performance "
"Trending and Analysis ...".format(count))
for plot in element["plots"]:
# Add layout to the plots:
layout = plot.get("layout", None)
if layout is not None:
try:
plot["layout"] = \
self.configuration["plot-layouts"][layout]
except KeyError:
raise PresentationError(
"Layout {0} is not defined in the "
"configuration section.".format(layout))
# Add data sets:
if isinstance(plot.get("data", None), str):
data_set = plot["data"]
try:
plot["data"] = \
self.configuration["data-sets"][data_set]
except KeyError:
raise PresentationError(
"Data set {0} is not defined in "
"the configuration section.".format(data_set))
self._specification["cpta"] = element
count += 1
logging.info("Done.")
def _parse_configuration(self):
"""Parse configuration of PAL in the specification YAML file.
"""
logging.info("Parsing specification file: configuration ...")
idx = self._get_type_index("configuration")
if idx is None:
logging.warning(
"No configuration information in the specification "
"file.")
return None
try:
self._specification["configuration"] = self._cfg_yaml[idx]
except KeyError:
raise PresentationError("No configuration defined.")
# Data sets: Replace ranges by lists
for set_name, data_set in self.configuration["data-sets"].items():
if not isinstance(data_set, dict):
continue
for job, builds in data_set.items():
if builds:
if isinstance(builds, dict):
build_end = builds.get("end", None)
try:
build_end = int(build_end)
except ValueError:
# defined as a range <start, build_type>
build_end = self._get_build_number(job, build_end)
builds = [
x for x in range(builds["start"], build_end + 1)
if x not in builds.get("skip", list())
]
self.configuration["data-sets"][set_name][job] = builds
elif isinstance(builds, list):
for idx, item in enumerate(builds):
try:
builds[idx] = int(item)
except ValueError:
# defined as a range <build_type>
builds[idx] = self._get_build_number(job, item)
# Data sets: add sub-sets to sets (only one level):
for set_name, data_set in self.configuration["data-sets"].items():
if isinstance(data_set, list):
new_set = dict()
for item in data_set:
try:
for key, val in self.configuration["data-sets"][item].\
items():
new_set[key] = val
except KeyError:
raise PresentationError(
"Data set {0} is not defined in "
"the configuration section.".format(item))
self.configuration["data-sets"][set_name] = new_set
# Mapping table:
mapping = None
mapping_file_name = self._specification["configuration"].\
get("mapping-file", None)
if mapping_file_name:
logging.debug("Mapping file: '{0}'".format(mapping_file_name))
try:
with open(mapping_file_name, 'r') as mfile:
mapping = load(mfile)
logging.debug("Loaded mapping table:\n{0}".format(mapping))
except (YAMLError, IOError) as err:
raise PresentationError(
msg="An error occurred while parsing the mapping file "
"'{0}'.".format(mapping_file_name),
details=repr(err))
# Make sure everything is lowercase
if mapping:
self._specification["configuration"]["mapping"] = \
{key.lower(): val.lower() for key, val in mapping.iteritems()}
else:
self._specification["configuration"]["mapping"] = dict()
# Ignore list:
ignore = None
ignore_list_name = self._specification["configuration"].\
get("ignore-list", None)
if ignore_list_name:
logging.debug("Ignore list file: '{0}'".format(ignore_list_name))
try:
with open(ignore_list_name, 'r') as ifile:
ignore = load(ifile)
logging.debug("Loaded ignore list:\n{0}".format(ignore))
except (YAMLError, IOError) as err:
raise PresentationError(
msg="An error occurred while parsing the ignore list file "
"'{0}'.".format(ignore_list_name),
details=repr(err))
# Make sure everything is lowercase
if ignore:
self._specification["configuration"]["ignore"] = \
[item.lower() for item in ignore]
else:
self._specification["configuration"]["ignore"] = list()
logging.info("Done.")
def _parse_elements(self):
"""Parse elements (tables, plots) specification in the specification
YAML file.
"""
logging.info("Parsing specification file: elements ...")
count = 1
for element in self._cfg_yaml:
try:
element["output-file"] = self._replace_tags(
element["output-file"],
self._specification["environment"]["paths"])
except KeyError:
pass
# add data sets to the elements:
if isinstance(element.get("data", None), str):
data_set = element["data"]
try:
element["data"] = self.configuration["data-sets"][data_set]
except KeyError:
raise PresentationError(
"Data set {0} is not defined in "
"the configuration section.".format(data_set))
if element["type"] == "table":
logging.info(" {:3d} Processing a table ...".format(count))
try:
element["template"] = self._replace_tags(
element["template"],
self._specification["environment"]["paths"])
except KeyError:
pass
self._specification["tables"].append(element)
count += 1
elif element["type"] == "plot":
logging.info(" {:3d} Processing a plot ...".format(count))
# Add layout to the plots:
layout = element["layout"].get("layout", None)
if layout is not None:
element["layout"].pop("layout")
try:
for key, val in (self.configuration["plot-layouts"]
[layout].items()):
element["layout"][key] = val
except KeyError:
raise PresentationError(
"Layout {0} is not defined in "
"the configuration section.".format(layout))
self._specification["plots"].append(element)
count += 1
elif element["type"] == "file":
logging.info(" {:3d} Processing a file ...".format(count))
try:
element["dir-tables"] = self._replace_tags(
element["dir-tables"],
self._specification["environment"]["paths"])
except KeyError:
pass
self._specification["files"].append(element)
count += 1
logging.info("Done.")
