def __init__(self, config):
"""
like all the the other inits around this code, the init will gather
relevant information for this class and put them into local shortcuts
:param config: The global promoter config or the reduced dlrnclient
config
"""
self.config = config
# TODO(gcerami): fix credentials gathering
dlrnapi_client.configuration.password = self.config.dlrnauth_password
dlrnapi_client.configuration.username = self.config.dlrnauth_username
api_client = dlrnapi_client.ApiClient(host=self.config.api_url)
self.api_instance = dlrnapi_client.DefaultApi(api_client=api_client)
self.last_promotions = {}
# Variable to detect changes on the hash while we are running a
# promotion
self.named_hashes_map = {}
# This way of preparing parameters and configuration is copied
# directly from dlrnapi CLI and ansible module
self.hashes_params = dlrnapi_client.PromotionQuery()
self.jobs_params = dlrnapi_client.Params2()
self.jobs_params_aggregate = dlrnapi_client.Params3()
self.report_params = dlrnapi_client.Params3()
self.promote_params = dlrnapi_client.Promotion()
self.log.debug("Promoter DLRN client: API URL: {}, user: {}"
"".format(api_client.host,
self.config.dlrnauth_username))
def get_dlrn_instance(config=None):
if config is None:
return None
api_client = dlrnapi_client.ApiClient(host=config.get('main', 'api_url'))
return dlrnapi_client.DefaultApi(api_client=api_client)
def check_dlrn_promoted_hash(stage_info):
''' Check that the commit, distro hash has been promoted to
promotion_target as recorded in DLRN. '''
dlrn_host = stage_info['dlrn_host']
promotion_target = stage_info['promotion_target']
commit_hash = stage_info['promotions']['promotion_candidate'][
'commit_hash']
distro_hash = stage_info['promotions']['promotion_candidate'][
'distro_hash']
logger = logging.getLogger('TestPromoter')
api_client = dlrnapi_client.ApiClient(host=dlrn_host)
dlrn = dlrnapi_client.DefaultApi(api_client=api_client)
params = dlrnapi_client.PromotionQuery()
params.commit_hash = commit_hash
params.distro_hash = distro_hash
try:
api_response = dlrn.api_promotions_get(params)
logger.debug(api_response)
except dlrnapi_client.rest.ApiException:
logger.error('Exception when calling api_promotions_get: %s',
dlrnapi_client.rest.ApiException)
raise
error_message = ("Expected commit hash: {}"
" has not been promoted to {}."
"".format(commit_hash, promotion_target))
conditions = [(promotion.promote_name == promotion_target)
for promotion in api_response]
assert any(conditions), error_message
def main():
module = AnsibleModule(argument_spec=dict(
action=dict(required=True,
choices=[
'repo-get', 'repo-use', 'repo-status', 'report-result',
'repo-promote', 'commit-import', 'promotion-get',
'build-metrics'
]),
host=dict(required=True),
user=dict(),
password=dict(no_log=True),
reporting_job_id=dict(),
max_age=dict(default='0', type='int'),
success=dict(type='bool'),
job_id=dict(),
sequential_mode=dict(type='bool', default=False),
previous_job_id=dict(),
commit_hash=dict(),
distro_hash=dict(),
info_url=dict(),
timestamp=dict(),
notes=dict(),
promote_name=dict(),
repo_url=dict(),
start_date=dict(),
end_date=dict(),
package_name=dict(),
))
action = module.params['action']
username = module.params['user']
password = module.params['password']
if action in auth_required_actions and (username is None
or password is None):
module.fail_json(msg="Action %s requires authentication" % action)
api_client = dlrnapi_client.ApiClient(host=module.params['host'])
dlrnapi_client.configuration.username = username
dlrnapi_client.configuration.password = password
api_instance = dlrnapi_client.DefaultApi(api_client=api_client)
options = DLRNAPIWrapper(module.params)
options.check_options(action, module)
try:
output = command_funcs[action](api_instance, options)
if type(output) == list:
output_f = [x.to_dict() for x in output]
else:
output_f = output.to_dict()
module.exit_json(changed=True, result=output_f)
except ApiException as e:
module.fail_json(msg="Exception when calling "
"%s: %s\n" % (command_funcs[action], e))
def promoter(config):
logger = logging.getLogger('promoter')
release = config.get('main', 'release')
get_lock('promoter')
logger.info('STARTED promotion process for release: %s', release)
try:
git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'],
cwd=os.path.abspath(sys.path[0]))
logger.info(
'Current git hash of repo containing the promoter '
'script: %s', git_hash.strip())
except OSError:
logger.debug('Failed to get the current git repo hash, check if '
'git is installed.')
except subprocess.CalledProcessError:
logger.debug('Failed to get the current git repo hash, probably not '
'running inside a git repository.')
# setup the API connection
dry_run = config.getboolean('main', 'dry_run')
api_client = dlrnapi_client.ApiClient(host=config.get('main', 'api_url'))
dlrnapi_client.configuration.username = config.get('main', 'username')
if os.getenv('DLRNAPI_PASSWORD', None) is None:
logger.warning('DLRNAPI_PASSWORD env variable is missing or empty, '
'promotion attempt will fail!')
dlrnapi_client.configuration.password = os.getenv('DLRNAPI_PASSWORD', None)
api_instance = dlrnapi_client.DefaultApi(api_client=api_client)
latest_hashes_count = config.getint('main', 'latest_hashes_count')
config.remove_section('main')
logger.info('Using API URL: %s', api_client.host)
# load the promote_from data
promote_from = {k: v for k, v in config.items('promote_from')}
logger.debug('Attempting to promote these DLRN links: %s', promote_from)
config.remove_section('promote_from')
# load the promotion requirements
job_reqs = {}
sections = config.sections()
for section in sections:
job_reqs[section] = [k for k, v in config.items(section)]
logger.debug('Promotion requirements loaded: %s', job_reqs)
promote_all_links(api_instance, promote_from, job_reqs, dry_run, release,
latest_hashes_count)
logger.info("FINISHED promotion process")
def check_dlrn_promoted_hash(stage_info=None, **kwargs):
"""
Check that the the supposed hash has been promoted to
promotion_target as recorded in DLRN.
:param stage_info: a dictionary containing parameter of the staging env
:param kwargs: additional parameter for non-staged executions
:return: None
"""
if stage_info is not None:
# We are checking a stage
api_url = stage_info['dlrn']['server']['api_url']
promotion_target = stage_info['dlrn']['promotion_target']
candidate_commit = \
stage_info['dlrn']['promotions']['promotion_candidate']
candidate_hash = DlrnHash(source=candidate_commit)
api_client = dlrnapi_client.ApiClient(host=api_url)
dlrn_client = dlrnapi_client.DefaultApi(api_client=api_client)
params = dlrnapi_client.PromotionQuery()
params.limit = 1
params.promote_name = promotion_target
else:
# We are checking production server
# TODO(gcerami) implement this branch ?
pass
try:
api_response = dlrn_client.api_promotions_get(params)
log.debug(api_response)
except dlrnapi_client.rest.ApiException:
log.error('Exception when calling api_promotions_get: %s',
dlrnapi_client.rest.ApiException)
raise
error_msg = "No promotions for hash {}".format(candidate_hash)
assert api_response != [], error_msg
promotion_hash = DlrnHash(source=api_response[0])
error_message = ("Expected full hash: {}"
" has not been promoted to {}."
"".format(promotion_hash.full_hash, promotion_target))
conditions = [(promotion.promote_name == promotion_target)
for promotion in api_response]
assert any(conditions), error_message
def get_promotions(release, promote_name):
host = get_endpoint(release)
api_client = dlrnapi_client.ApiClient(host=host)
api_instance = dlrnapi_client.DefaultApi(api_client=api_client)
params = dlrnapi_client.PromotionQuery()
if promote_name:
params.promote_name = promote_name
try:
api_response = api_instance.api_promotions_get(params)
except ApiException as e:
print("Exception when calling DefaultApi->api_promotions_get: %s\n" %
e)
return api_response
def main():
parser = argparse.ArgumentParser(prog='dlrnapi')
parser.add_argument('--url', required=True, help='URL to use')
parser.add_argument('--username',
'-u',
help='username for authentication, defaults to '
'"DLRNAPI_USERNAME" environment variable if set',
default=os.getenv('DLRNAPI_USERNAME', None))
parser.add_argument('--password',
'-p',
help='password for authentication, defaults to '
'"DLRNAPI_PASSWORD" environment variable is set',
default=os.getenv('DLRNAPI_PASSWORD', None))
subparsers = parser.add_subparsers(dest='command',
title='subcommands',
description='available subcommands')
# Subcommand get-repo
parser_last = subparsers.add_parser('repo-get',
help='Get last tested repo')
parser_last.add_argument('--max-age', type=int, default=0, help='max_age')
parser_last.add_argument('--success',
type=str,
default=None,
help='Find repos with a successful/unsuccessful '
'vote, if true or false are specified')
parser_last.add_argument('--job-id',
type=str,
default=None,
help='Name of the CI that sent the vote. If not '
'set, no filter will be set on CI')
parser_last.add_argument('--sequential-mode',
dest='sequential',
action='store_true',
help='Use the sequential mode algorithm. In this '
'case, return the last tested repo within '
'that timeframe for the CI job described by '
'--previous-job-id')
parser_last.set_defaults(sequential=False)
parser_last.add_argument('--previous-job-id',
type=str,
default=None,
help='If --sequential-mode is set, look for jobs'
' tested by this CI')
# Subcommand use-repo
parser_use_last = subparsers.add_parser('repo-use',
help='Get the last tested repo '
'since a specific time '
'(optionally for a CI job), '
'and add an "in progress" '
'entry in the CI job table '
'for this.')
parser_use_last.add_argument('--max-age',
type=int,
default=0,
help='max_age')
parser_use_last.add_argument('--reporting-job-id',
type=str,
required=True,
help=' Name of the CI that will add the "in '
'progress" entry in the CI job table.')
parser_use_last.add_argument('--success',
type=str,
default=None,
help='Find repos with a successful/'
'unsuccessful vote, if true or false '
'are specified')
parser_use_last.add_argument('--job-id',
type=str,
default=None,
help='Name of the CI that sent the vote. If '
'not set, no filter will be set on CI')
parser_use_last.add_argument('--sequential-mode',
dest='sequential',
action='store_true',
help='Use the sequential mode algorithm. In '
'this case, return the last tested repo '
'within that timeframe for the CI job '
'described by --previous-job-id')
parser_use_last.set_defaults(sequential=False)
parser_use_last.add_argument('--previous-job-id',
type=str,
default=None,
help='If --sequential-mode is true, look for '
'jobs tested by this CI')
# Subcommand repo-status
parser_st = subparsers.add_parser('repo-status',
help='Get all the CI reports for a '
'specific repository.')
parser_st.add_argument('--commit-hash',
type=str,
required=True,
help='commit_hash of the repo to fetch '
'information for.')
parser_st.add_argument('--distro-hash',
type=str,
required=True,
help='distro_hash of the repo to fetch '
'information for.')
parser_st.add_argument('--success',
type=str,
default=None,
help='If set to a value (true/false), only return '
'the CI reports with the specified vote. If '
'not set, return all CI reports.')
# Subcommand report-result
parser_rep = subparsers.add_parser('report-result',
help='Report the result of a CI job')
parser_rep.add_argument('--job-id',
type=str,
required=True,
help='Name of the CI sending the vote')
parser_rep.add_argument('--commit-hash',
type=str,
required=True,
help='commit_hash of tested repo')
parser_rep.add_argument('--distro-hash',
type=str,
required=True,
help='distro_hash of tested repo')
parser_rep.add_argument('--info-url',
type=str,
required=True,
help='URL where to find additional information '
'from the CI execution')
parser_rep.add_argument('--timestamp',
type=str,
required=True,
help='Timestamp (in seconds since the epoch)')
parser_rep.add_argument('--success',
type=str,
required=True,
help='Was the CI execution successful? Set to '
'true or false.')
parser_rep.add_argument('--notes', type=str, help='Additional notes')
# Subcommand promote
parser_prom = subparsers.add_parser('repo-promote',
help='Promote a repository')
parser_prom.add_argument('--commit-hash',
type=str,
required=True,
help='commit_hash of the repo to be promoted')
parser_prom.add_argument('--distro-hash',
type=str,
required=True,
help='distro_hash of the repo to be promoted')
parser_prom.add_argument('--promote-name',
type=str,
required=True,
help='Name to be used for the promotion')
# Subcommand promotion-get
parser_promget = subparsers.add_parser('promotion-get',
help='Get information about '
'promotions')
parser_promget.add_argument('--commit-hash',
type=str,
required=False,
help='commit_hash of the repo to search '
'promotions for. Requires --distro-hash '
'if specified.')
parser_promget.add_argument('--distro-hash',
type=str,
required=False,
help='distro_hash of the repo to search '
'promotions for. Requires --commit-hash '
'if specified.')
parser_promget.add_argument('--promote-name',
type=str,
required=False,
help='Filter results for this promotion name.')
parser_promget.add_argument('--offset',
type=int,
required=False,
help='Show results after this offset. Each '
'query will only return 100 entries by '
'default.')
parser_promget.add_argument('--limit',
type=int,
required=False,
help='Limit the results to the first limit '
'items')
# Subcommand commit-import
parser_imp = subparsers.add_parser('commit-import',
help='Import a commit built by another'
' instance')
parser_imp.add_argument('--repo-url',
type=str,
required=True,
help='Base repository URL for the remote repo '
'to import')
# Subcommand build-metrics
parser_metrics = subparsers.add_parser(
'build-metrics', help='Fetch build metrics in a time period')
parser_metrics.add_argument(
'--start-date',
type=str,
required=True,
help='Start date for the query, in YYYY-MM-DD format')
parser_metrics.add_argument(
'--end-date',
type=str,
required=True,
help='End date for the query, in YYYY-MM-DD format')
parser_metrics.add_argument(
'--package-name',
type=str,
required=False,
help='If specified, only fetch metrics for this package name')
options, args = parser.parse_known_args(sys.argv[1:])
# create an instance of the API class
api_client = dlrnapi_client.ApiClient(host=options.url)
dlrnapi_client.configuration.username = options.username
dlrnapi_client.configuration.password = options.password
api_instance = dlrnapi_client.DefaultApi(api_client=api_client)
try:
api_response = command_funcs[options.command](api_instance, options)
print(
json.dumps(api_response,
cls=ResponseEncoder,
indent=2,
sort_keys=True))
except ApiException as e:
# Handle 404 exceptions gracefully
if e.status == 404:
print(
"ERROR: Got error 404, probably endpoint %s is not available" %
options.url)
return 1
else:
raise
except Exception as e:
raise e
def promoter(config):
logger = logging.getLogger('promoter')
distro = (config.get('main', 'distro_name').lower(),
config.get('main', 'distro_version'))
release = config.get('main', 'release')
api_url = config.get('main', 'api_url')
# No boolean values here, as these are passed directly to ansible-playbook
# command line
manifest_push = config.get('main', 'manifest_push', fallback="false")
target_registries_push = config.get('main', 'target_registries_push',
fallback="true")
logger.info('STARTED promotion process for release: %s', release)
try:
git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'],
cwd=os.path.abspath(sys.path[0]))
logger.info('Current git hash of repo containing the promoter '
'script: %s', git_hash.strip())
except OSError:
logger.debug('Failed to get the current git repo hash, check if '
'git is installed.')
except subprocess.CalledProcessError:
logger.debug('Failed to get the current git repo hash, probably not '
'running inside a git repository.')
# setup the API connection
dry_run = config.getboolean('main', 'dry_run')
api_client = dlrnapi_client.ApiClient(host=config.get('main', 'api_url'))
dlrnapi_client.configuration.username = config.get('main', 'username')
if os.getenv('DLRNAPI_PASSWORD', None) is None:
logger.warning('DLRNAPI_PASSWORD env variable is missing or empty, '
'promotion attempt will fail!')
dlrnapi_client.configuration.password = os.getenv('DLRNAPI_PASSWORD', None)
api_instance = dlrnapi_client.DefaultApi(api_client=api_client)
latest_hashes_count = config.getint('main', 'latest_hashes_count')
config.remove_section('main')
logger.info('Using API URL: %s', api_client.host)
# load the promote_from data
promote_from = {k: v for k, v in config.items('promote_from')}
logger.debug('Attempting to promote these DLRN links: %s',
promote_from)
config.remove_section('promote_from')
# load the promotion requirements
job_reqs = {}
sections = config.sections()
for section in sections:
job_reqs[section] = [k for k, v in config.items(section)]
logger.debug('Promotion requirements loaded: %s', job_reqs)
global start_named_hashes
start_named_hashes = fetch_current_named_hashes(
release, promote_from, api_instance)
logger.info('Named hashes at start of promotion process: %s',
start_named_hashes)
promote_all_links(
api_instance,
promote_from,
job_reqs,
dry_run,
distro,
release,
latest_hashes_count,
api_url,
manifest_push,
target_registries_push)
logger.info("FINISHED promotion process")
for status in api_response:
if wait_job is None and status.job_id == wait_job_name:
wait_job = status
elif launch_job is None and status.job_id == launch_job_name:
launch_job = status
logger.info("Selected wait job build: {}".format(wait_job))
logger.info("Selected launch job build: {}".format(launch_job))
# Trigger if job is not already trigger and wait job is fine
return launch_job is None and wait_job is not None and wait_job.success
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description="Check condition at DLRN to trigger job")
parser.add_argument(
'--dlrn-url',
default="https://trunk.rdoproject.org/api-centos-master-uc")
parser.add_argument('--promotion-name', default="tripleo-ci-testing")
parser.add_argument(
'--wait-job',
default="periodic-tripleo-centos-7-master-containers-build")
parser.add_argument('--launch-job', default="test-connection-to-hardware")
args = parser.parse_args()
client = dlrnapi_client.ApiClient(host=args.dlrn_url)
dlrn = dlrnapi_client.DefaultApi(api_client=client)
if not check_trigger_condition(dlrn, args.promotion_name, args.wait_job,
args.launch_job):
sys.exit(1)
def get_dlrn_client(config):
api_client = dlrnapi_client.ApiClient(host=config['main']['api_url'])
return dlrnapi_client.DefaultApi(api_client=api_client)