def vote(self, dlrn_hash, job_id, job_url, vote):
"""
Add a CI vote for a job for a certain hash
This method is used mainly in staging environment to create basic
promotions to handle
:param dlrn_hash: The hash with the info for promotion
:param job_id: The name of the job that votes
:param job_url: The url of the job that votes
:param vote: A bool representing success(true) or failure(false)
:return: the API response after voting
"""
params = copy.deepcopy(self.report_params)
if type(dlrn_hash) == DlrnCommitDistroHash:
dlrn_hash.dump_to_params(params)
elif type(dlrn_hash) == DlrnAggregateHash:
# votes for the aggregate hash cannot contain commit and distro
params.aggregate_hash = dlrn_hash.aggregate_hash
params.success = str(vote)
params.timestamp = dlrn_hash.timestamp
params.job_id = job_id
params.url = job_url
# Remove annoying new lines in params __str__
str_params = str(params).replace('\n', ' ').replace('\r', ' ')
self.log.debug("Dlrn voting success: %s for dlrn_hash %s"
"", params.success, dlrn_hash)
self.log.info(
"Dlrn voting success: %s for job %s with parameters %s"
"", params.success, job_id, str_params)
try:
api_response = self.api_instance.api_report_result_post(params)
self.log.info("Dlrn voted success: %s for job %s on hash %s"
"", params.success, job_id, dlrn_hash)
except ApiException as ae:
message = ae.body
try:
body = json.loads(ae.body)
message = body['message']
except JSONDecodeError:
pass
self.log.error(
"Dlrn voting success: %s for dlrn_hash %s: Error "
"during voting through API: (%s) %s: %s"
"", params.success, dlrn_hash, ae.status, ae.reason, message)
self.log.error("------- -------- Promoter aborted")
raise ae
if not api_response:
self.log.error(
"Dlrn voting success: %s for dlrn_hash %s: API "
"vote response is empty"
"", params.success, dlrn_hash)
self.log.error("------- -------- Promoter aborted")
raise PromotionError("Dlrn Vote failed")
return api_response
def promote(self,
dlrn_hash,
target_label,
candidate_label=None,
create_previous=True):
"""
This method prepares the promotion environment for the hash.
It creates a previous promotion link if it's requested. and orchestrate
the calls to the actual promotion function
:param dlrn_hash: The hash to promote
:param target_label: The name to promote the hash to
:param candidate_label: The name the hash was recently promoted to,
if any
:param create_previous: A bool value to define if we want to also
create a previous link for the previous hash value of target_label
:return: None
"""
incumbent_hash = self.fetch_promotions(target_label, count=1)
if incumbent_hash and incumbent_hash == dlrn_hash:
# If this happens, something went horribly wrong. We are
# trying to promote again something that was already
# promoted, and for some reason, all the levels of check up to this
# failed.
self.log.critical(
"Dlrn promote: hash %s seems to already have "
"been promoted to %s, and all code checks to "
"avoid this at this point failed. Check the "
"code.", dlrn_hash, target_label)
raise PromotionError("Attempted to promote an already promoted "
"hash")
log_header = ("Dlrn promote '{}' from {} to {}:"
"".format(dlrn_hash, candidate_label, target_label))
if create_previous:
# Save current hash as previous-$link
if incumbent_hash:
previous_target_label = "previous-" + target_label
incumbent_hash.label = target_label
self.log.info("%s moving previous promoted hash '%s' to %s"
"", log_header, incumbent_hash,
previous_target_label)
self.promote_hash(log_header,
incumbent_hash,
previous_target_label,
candidate_label=target_label)
else:
self.log.warning("%s No previous promotion found", log_header)
self.log.info("%s Attempting promotion", log_header)
self.promote_hash(log_header,
dlrn_hash,
target_label,
candidate_label=candidate_label)
def get_hash_from_component(self, log_header, component_name, base_url):
"""
Downloads the commit.yaml file relative to the component, and creates
an hash with all the information in it
:param log_header: the header for all logging messages
:param component_name: the name of the component
:param base_url: The base url taken from the component in the main
delorean.repo file
:return: A DlrnDistroCommitHash containing the information of the
promoted component
"""
self.log.debug("%s base url url for component %s at %s", log_header,
component_name, base_url)
commit_url = "{}/{}".format(base_url, "commit.yaml")
self.log.debug("%s commit info url for component %s at %s", log_header,
component_name, commit_url)
try:
# FIXME: in python2 urlopen is not a context manager
with contextlib.closing(url.urlopen(commit_url)) as commits_yaml:
commits = yaml.safe_load(commits_yaml.read().decode("UTF-8"))
# FIXME(gcerami) it is very difficult to make urlopen generate
# url.HTTPError (without mocking side effect directly), so this part
# is only partially tested
except (url.HTTPError, url.URLError):
self.log.error(
"Dlrn Promote: Error downloading component yaml info"
" at %s", commit_url)
self.log.error("------- -------- Promoter aborted")
raise PromotionError("Unable to fetch commits from component url")
# AP step4: from commits.yaml extract commit/distro_hash to
# promote and create an Hash object
promotion_info = commits['commits'][0]
promotion_info['timestamp'] = promotion_info['dt_commit']
self.log.debug("%s component '%s' commit info: %s", log_header,
component_name, promotion_info)
# AP step5: add hashes to promotion list
promotion_hash = DlrnCommitDistroHash(source=promotion_info)
self.log.debug("%s adding '%s' to promotion list", log_header,
promotion_hash)
return promotion_hash
def promote(self, candidate_hash, target_label, candidate_label=None):
"""
This method promotes containers whose tag is equal to the dlrn_id
specified by retagging them with the target_label.
Currently invokes an external ansible playbook for the effective
promotion
:param candidate_hash: The hash to select container tags
:param target_label: the new tag to apply to the containers
:param kwarg: unused
:return: None
"""
self.log.info("Containers promote '{}' to {}: Attempting promotion"
"".format(candidate_hash, target_label))
extra_vars_path = self.prepare_extra_vars(candidate_hash, target_label,
candidate_label)
# Use single string to make it easy to copy/paste from logs
cmd = ("env "
"ANSIBLE_LOG_PATH={} "
"ANSIBLE_DEBUG=False "
"ansible-playbook -v -e @{} {}".format(
self.logfile,
extra_vars_path,
self.promote_playbook,
))
log_header = "Containers promote '{}' to {}:".format(
candidate_hash, target_label)
# Remove color codes from ansible output
ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]')
try:
self.log.info('Running: %s', cmd)
container_logs = subprocess.check_output(cmd.split(" "),
stderr=subprocess.STDOUT)
# containers_log needs decoding in python3
except subprocess.CalledProcessError as ex:
os.unlink(extra_vars_path)
self.log.error("%s Failed promotion", log_header)
self.log.error(
"%s Failed promotion"
"start logs -----------------------------", log_header)
for line in ex.output.decode("UTF-8").split("\n"):
self.log.error(ansi_escape.sub('', line))
self.log.exception(ex)
self.log.error(
"%s Failed promotion end"
" logs -----------------------------", log_header)
raise PromotionError("Failed to promote overcloud images")
os.unlink(extra_vars_path)
if type(container_logs) is not str:
container_logs = container_logs.decode()
self.log.info("%s Successful promotion", log_header)
self.log.debug(
"%s Successful "
"promotion start logs -----------------------------", log_header)
for line in container_logs.split("\n"):
self.log.debug(ansi_escape.sub('', line))
self.log.debug(
"%s Successful "
"promotion end logs -----------------------------", log_header)
def promote(self,
candidate_hash,
target_label,
candidate_label=None,
create_previous=True,
validation=True):
"""
Effective promotion of the images. This method will handle symbolic
links to the dir containing images from the candidate hash,
optionally saving the current link as previous
:param candidate_hash: The dlrn hash to promote
:param target_label: The name of the link to create
:param candidate_label: Currently unused
:param create_previous: A bool to determine if previous link is created
:param validation: A bool to determine if qcow validation should be done
:return: None
"""
self.client.connect()
if validation:
self.validate_qcows(candidate_hash)
self.client.chdir(self.images_dir)
log_header = "Qcow promote '{}' to {}:".format(candidate_hash,
target_label)
self.log.info("%s Attempting promotion", log_header)
# Check if candidate_hash dir is present
try:
self.client.stat(candidate_hash.full_hash)
except EnvironmentError as ex:
self.log.error(
"%s images dir for hash %s not present or not "
"accessible", log_header, candidate_hash)
self.log.exception(ex)
self.client.close()
raise PromotionError("{} No images dir for hash {}"
"".format(log_header, candidate_hash))
# Check if the target label exists and points to a hash dir
current_hash = None
try:
current_hash = self.client.readlink(target_label)
except EnvironmentError:
self.log.debug("%s No link named %s exists", log_header,
target_label)
# If this exists Check if we can remove the symlink
if current_hash:
self.rollback_links['target_label'] = current_hash
try:
self.client.remove(target_label)
except EnvironmentError as ex:
self.log.debug("Unable to remove the target_label: %s",
target_label)
self.log.exception(ex)
self.client.close()
raise
# Check if a previous link exists and points to an hash-dir
previous_label = "previous-{}".format(target_label)
previous_hash = None
try:
previous_hash = self.client.readlink(previous_label)
except EnvironmentError:
self.log.debug("%s No previous-link named %s exists", log_header,
previous_label)
self.log.debug("Previous hash %s", previous_hash)
# If it exists and we are handling it, check if we can remove and
# reassign it
if current_hash and previous_hash and create_previous:
self.rollback_links[previous_label] = previous_hash
try:
self.client.remove(previous_label)
except EnvironmentError as ex:
self.log.debug("Unable to remove the target_label: %s",
target_label)
self.log.exception(ex)
self.client.close()
# Rollback is not tested, we enable it later, when tests are
# easier to add
# self.rollback()
raise
try:
self.client.symlink(current_hash, previous_label)
except EnvironmentError as ex:
self.log.error("%s failed to link %s to %s", log_header,
previous_label, current_hash)
self.log.exception(ex)
# Rollback is not tested, we enable it later, when tests are
# easier to add
# self.rollback()
self.client.close()
raise
# Finally the effective promotion
try:
self.client.symlink(candidate_hash.full_hash, target_label)
except EnvironmentError as ex:
self.log.error("%s failed to link %s to %s", log_header,
target_label, candidate_hash.full_hash)
self.log.exception(ex)
# Rollback is not tested, we enable it later, when tests are
# easier to add
# self.rollback()
finally:
self.client.close()
self.log.info("%s Successful promotion", log_header)
def promote_hash(self,
log_header,
dlrn_hash,
target_label,
candidate_label=None):
"""
Select the promotion parameters creation method and the proper api
call to promote the type of hashes, then call the api.
:param log_header: Header of all logging message in this function
:param dlrn_hash: The dlrn hash to promote
:param target_label: The label/name to promote dlrn_hash to
:param candidate_label: The name/label the dlrn_hash was recently
promoted to, if any (mandatory for aggregate promotion)
:return: the promoted hash. It's the same hash for commitdistro,
and the last component promoted for the aggregate hash
"""
hash_type = type(dlrn_hash)
self.log.debug("%s promoting a %s", log_header, hash_type)
if hash_type is DlrnCommitDistroHash:
promotion_parameters = copy.deepcopy(self.promote_params)
dlrn_hash.dump_to_params(promotion_parameters)
promotion_parameters.promote_name = target_label
api_call = self.api_instance.api_promote_post
elif hash_type is DlrnAggregateHash:
promotion_parameters = \
self.get_promotion_aggregate_hashes(log_header,
dlrn_hash,
candidate_label,
target_label)
api_call = self.api_instance.api_promote_batch_post
else:
self.log.error("Unrecognized dlrn hash type: %s", hash_type)
raise PromotionError("Unknown hash type")
try:
promoted_info = api_call(promotion_parameters)
except ApiException as ae:
message = ae.body
try:
body = json.loads(ae.body)
message = body['message']
except JSONDecodeError:
pass
self.log.error(
"Exception while promoting hashes to API endpoint "
"(%s) %s: %s", ae.status, ae.reason, message)
self.log.error("------- -------- Promoter aborted")
raise ae
# The promoted info will sometimes return the aggregate,
# and always the timestamp
# but we'll always be interested in comparing just commit and
# distro hashes
stored_timestamp = dlrn_hash.timestamp
dlrn_hash.timestamp = None
promoted_hash = DlrnHash(source=promoted_info)
promoted_hash.timestamp = None
# Here we compare request and response. We check that info returned by
# the api call corresponds to the info passed to the api call.
# This seemingly stupid check already helped find at least 3 bugs
# in code and tests, because we think we are promoting something when
# we are promoting something completely different
if dlrn_hash == promoted_hash:
self.log.info("%s (subhash %s) Successfully promoted", log_header,
dlrn_hash)
else:
self.log.error(
"%s (subhash %s) API returned different"
" promoted hash: '%s'", log_header, dlrn_hash, promoted_hash)
# Until we have a better way to compute the effective expectation
# on the promoted hash, don't make this fatal.
# raise PromotionError("API returned different promoted hash")
# For every hash promoted, we need to update the named hashes.
self.update_current_named_hashes(dlrn_hash, target_label)
# Add back timestamp
dlrn_hash.timestamp = stored_timestamp
return dlrn_hash
def get_promotion_aggregate_hashes(self, log_header, dlrn_hash,
candidate_label, target_label):
"""
Return the hashes needed for the promotion of a commidistro hash
:param log_header: the header for all logging messages
:param dlrn_hash: The hash to use as a starting point
:param candidate_label: The name the hash was recently promoted to,
if any
:param target_label: The name to promote the hash to
:return: A list with all the hashes needed to promote, to have
the starting hash correctly promoted. In aggregate case these are all
the hashes from the component that form the aggregate hash
"""
promotion_parameters_list = []
# Aggregate hash cannot be promoted directly, we need to promote
# all the components the aggregate points to singularly
# Aggregate promotion step 1: download the full delorean repo
# and save it locally for parsing
candidate_url = ("{}/{}/delorean.repo"
"".format(self.config.repo_url, dlrn_hash.commit_dir))
self.log.debug("Dlrn promote '%s': URL for candidate label repo: %s",
dlrn_hash, candidate_url)
try:
# FIXME: in python2 urlopen is not a context manager
with contextlib.closing(
url.urlopen(candidate_url)) as remote_repo_content:
# FIXME: in python2 configparser can read a config only from
# a file or a file-like obj. But python3 need the file
# to be converted first in UTF-8
remote_repo_content = remote_repo_content.read().decode()
except (url.HTTPError, url.URLError):
self.log.error(
"Dlrn Promote: Error downloading delorean repo"
" at %s", candidate_url)
self.log.error("------- -------- Promoter aborted")
raise PromotionError("Unable to fetch repo from repo url at %s",
candidate_url)
# Tried to use stringIO here, but the config.readfp seems not
# to be working correctly with stringIO, so a temporary file
# is needed
__, repo_file_path = tempfile.mkstemp()
repo_config = ini_parser.ConfigParser()
with open(repo_file_path, "w+") as repo_file:
repo_file.write(remote_repo_content)
repo_file.seek(0)
# FIXME python3 can read from a string, doesn't need a fp
repo_config.readfp(repo_file)
os.unlink(repo_file_path)
# AP step2: for all the subrepos in repo file get the baseurl for
# all the components
components = repo_config.sections()
if not components:
self.log.error("%s aggregate repo at %s contains no components",
log_header, candidate_url)
self.log.error("------- -------- Promoter aborted")
raise PromotionError("DLRN aggregate repo is empty")
else:
self.log.info("%s aggregate repo at %s contains components %s",
log_header, candidate_url, ', '.join(components))
# AP step3 download commits information for all the single
# component
for component_name in components:
base_url = repo_config.get(component_name, 'baseurl')
promotion_hash = self.get_hash_from_component(
log_header, component_name, base_url)
promotion_parameters = copy.deepcopy(self.promote_params)
promotion_hash.dump_to_params(promotion_parameters)
promotion_parameters.promote_name = target_label
promotion_parameters_list.append(promotion_parameters)
return sorted(promotion_parameters_list, key=lambda x: x.timestamp)