def test_template_build_type(image_url, expected_type):
"""test the build_type parameter of TemplateName class
Bogus build_url, only image_url is used for this property
TODO: mock requests for testing other parameters of TemplateName
"""
t = TemplateName(build_url='http://fake.example.com', image_url=image_url)
assert t.build_type == expected_type
def test_template_parsing(tmp_name, info_tuple):
parsed = TemplateName.parse_template(tmp_name)
print('template name: {}'.format(tmp_name))
print('Parsed template: {}'.format(parsed))
print('expected: {}'.format(info_tuple))
diff = DeepDiff(
parsed,
info_tuple,
verbose_level=0, # If any higher, will flag string vs unicode
ignore_order=False)
assert diff == {}
def main():
urls = cfme_data['basic_info']['cfme_images_url']
stream = args.stream or cfme_data['template_upload']['stream']
upload_url = args.image_url
provider_type = args.provider_type or cfme_data['template_upload']['provider_type']
if args.provider_data is not None:
local_datafile = open(args.provider_data, 'r').read()
create_datafile = open(path.conf_path.strpath + '/provider_data.yaml', 'w')
create_datafile.write(local_datafile)
create_datafile.close()
provider_data = cfme.utils.conf.provider_data
stream = provider_data['stream']
if stream:
urls = {}
image_url = cfme_data['basic_info']['cfme_images_url']
urls[stream] = image_url.get(stream)
if not urls[stream]:
image_url = cfme_data['basic_info']['cfme_old_images_url']
urls[stream] = image_url.get(stream)
if not urls[stream]:
base_url = cfme_data['basic_info']['cfme_old_images_url']['base_url']
version = ''.join(re.findall(r'(\d+)', stream))
urls[stream] = \
base_url + '.'.join(version[:2]) + '/' + '.'.join(version) + '/'
for key, url in urls.items():
if stream is not None:
if key != stream:
continue
if upload_url:
# strip trailing slashes just in case
if url.rstrip('/') != upload_url.rstrip('/'):
continue
dir_files = browse_directory(url)
if not dir_files:
continue
checksum_url = url + "SHA256SUM"
try:
urlopen(checksum_url)
except Exception:
logger.exception("No valid checksum file for %r, Skipping", key)
continue
kwargs = {}
module = None
if not provider_type:
sys.exit('specify the provider_type')
if provider_type == 'openstack':
module = 'template_upload_rhos'
if module not in dir_files:
continue
elif provider_type == 'rhevm':
module = 'template_upload_rhevm'
if module not in dir_files:
continue
elif provider_type == 'virtualcenter':
module = 'template_upload_vsphere'
if module not in dir_files:
continue
elif provider_type == 'scvmm':
module = 'template_upload_scvmm'
if module not in dir_files:
continue
elif provider_type == 'gce':
module = 'template_upload_gce'
if module not in dir_files:
continue
elif provider_type == 'ec2':
module = 'template_upload_ec2'
if module not in dir_files:
continue
elif provider_type == 'openshift':
module = 'template_upload_openshift'
if module not in dir_files:
continue
if not module:
logger.error('Could not match module to given provider type')
return 1
kwargs['stream'] = stream
kwargs['image_url'] = dir_files[module]
if args.provider_data is not None:
kwargs['provider_data'] = provider_data
else:
kwargs['provider_data'] = None
if cfme_data['template_upload']['automatic_name_strategy']:
kwargs['template_name'] = template_name(
dir_files[module],
dir_files[module + "_date"],
checksum_url,
get_version(url)
)
if not stream:
# Stream is none, using automatic naming strategy, parse stream from template name
template_parser = TemplateName.parse_template(kwargs['template_name'])
if template_parser.stream:
kwargs['stream'] = template_parser.group_name
if args.print_name_only:
print(kwargs['template_name'])
return 0
logger.info("TEMPLATE_UPLOAD_ALL:-----Start of %r upload on: %r--------",
kwargs['template_name'], provider_type)
logger.info("Executing %r with the following kwargs: %r", module, kwargs)
getattr(__import__(module), "run")(**kwargs)
logger.info("TEMPLATE_UPLOAD_ALL:------End of %r upload on: %r--------",
kwargs['template_name'], provider_type)
return 0
def main(trackerbot_url, mark_usable=None, selected_provider=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = (set(list_provider_keys())
if not selected_provider else set(selected_provider))
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
ipaddress = cfme_data.management_systems[provider_key].get('ipaddress')
if ipaddress and not net.is_pingable(ipaddress):
continue
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers,
unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [
pt['id'] for pt in trackerbot.depaginate(
api, api.providertemplate.get())['objects']
]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
template_info = TemplateName.parse_template(template_name)
# Don't want sprout templates
if template_info.group_name in ('sprout', 'rhevm-internal'):
logger.info('Ignoring %s from group %s', template_name,
template_info.group_name)
continue
seen_templates.add(template_name)
group = trackerbot.Group(template_info.group_name,
stream=template_info.stream)
try:
template = trackerbot.Template(template_name, group,
template_info.datestamp)
except ValueError:
logger.exception('Failure parsing provider %s template: %s',
provider_key, template_name)
continue
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name,
provider_key) in existing_provider_templates:
logger.info('Template %s already tracked for provider %s',
template_name, provider_key)
continue
try:
trackerbot.mark_provider_template(api, provider, template,
**usable)
logger.info(
'Added %s template %s on provider %s (datestamp: %s)',
template_info.group_name, template_name, provider_key,
template_info.datestamp)
except SlumberHttpBaseException:
logger.exception('%s: exception marking template %s', provider,
template)
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in trackerbot.depaginate(api,
api.providertemplate.get())['objects']:
key, template_name = pt['provider']['key'], pt['template']['name']
if key not in template_providers[
template_name] and key not in unresponsive_providers:
if key in all_providers:
logger.info("Cleaning up template %s on %s", template_name,
key)
trackerbot.delete_provider_template(api, key, template_name)
else:
logger.info(
"Skipping template cleanup %s on unknown provider %s",
template_name, key)
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(api, api.template.get())['objects']:
if not template['providers']:
logger.info("Deleting template %s (no providers)",
template['name'])
api.template(template['name']).delete()
image_url = cmd_args.image_url
stream = get_stream_from_image_url(image_url,
quiet=cmd_args.print_name_only)
elif cmd_args.stream and not cmd_args.image_url:
# default to base URL for given stream:
stream = cmd_args.stream
image_url = ALL_STREAMS[stream]
else:
logger.warning("Require either --image-url or --stream")
sys.exit(1)
provider_type = cmd_args.provider_type
template_name = ("{}-{}".format(cmd_args.template_name, stream)
if cmd_args.template_name else
TemplateName(image_url).template_name)
if cmd_args.print_name_only:
print(template_name)
sys.exit(0)
if not provider_type or cmd_args.provider:
provider_types = PROVIDER_TYPES
elif provider_type in PROVIDER_TYPES:
provider_types = [
provider_type,
]
else:
logger.error('Template upload for %r is not implemented yet.',
provider_type)
sys.exit(1)
def main(trackerbot_url, mark_usable=None, selected_provider=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = (set(list_provider_keys())
if not selected_provider
else set(selected_provider))
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
ipaddress = cfme_data.management_systems[provider_key].get('ipaddress')
if ipaddress and not net.is_pingable(ipaddress):
continue
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers, unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [
pt['id']
for pt
in trackerbot.depaginate(api, api.providertemplate.get())['objects']]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
template_info = TemplateName.parse_template(template_name)
# it turned out that some providers like ec2 may have templates w/o names.
# this is easy protection against such issue.
if not template_name.strip():
logger.warn('Ignoring template w/o name on provider %s', provider_key)
continue
# Don't want sprout templates
if template_info.group_name in ('sprout', 'rhevm-internal'):
logger.info('Ignoring %s from group %s', template_name, template_info.group_name)
continue
seen_templates.add(template_name)
group = trackerbot.Group(template_info.group_name, stream=template_info.stream)
try:
template = trackerbot.Template(template_name, group, template_info.datestamp)
except ValueError:
logger.exception('Failure parsing provider %s template: %s',
provider_key, template_name)
continue
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name, provider_key) in existing_provider_templates:
logger.info('Template %s already tracked for provider %s',
template_name, provider_key)
continue
try:
trackerbot.mark_provider_template(api, provider, template, **usable)
logger.info('Added %s template %s on provider %s (datestamp: %s)',
template_info.group_name,
template_name,
provider_key,
template_info.datestamp)
except SlumberHttpBaseException:
logger.exception('%s: exception marking template %s', provider, template)
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']:
key, template_name = pt['provider']['key'], pt['template']['name']
if key not in template_providers[template_name] and key not in unresponsive_providers:
if key in all_providers:
logger.info("Cleaning up template %s on %s", template_name, key)
trackerbot.delete_provider_template(api, key, template_name)
else:
logger.info("Skipping template cleanup %s on unknown provider %s",
template_name, key)
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(api, api.template.get())['objects']:
if not template['providers'] and template['name'].strip():
logger.info("Deleting template %s (no providers)", template['name'])
api.template(template['name']).delete()
def main(trackerbot_url, mark_usable=None, selected_provider=None, **kwargs):
tb_api = trackerbot.api(trackerbot_url)
all_providers = set(selected_provider or [
key for key, data in cfme_data.management_systems.items()
if 'disabled' not in data.get('tags', [])
])
bad_providers = manager.Queue()
# starmap the list of provider_keys into templates_on_provider
# return is list of ProvTemplate tuples
with ThreadPool(8) as pool:
mgmt_templates = pool.starmap(templates_on_provider,
((provider_key, bad_providers)
for provider_key in all_providers))
# filter out the misbehaving providers
bad_provider_keys = []
while not bad_providers.empty():
bad_provider_keys.append(bad_providers.get())
logger.warning('Filtering out providers that failed template query: %s',
bad_provider_keys)
working_providers = {
key
for key in all_providers if key not in bad_provider_keys
}
# Flip mgmt_templates into dict keyed on template name, listing providers
# [
# {prov1: [t1, t2]},
# {prov2: [t1, t3]},
# ]
#
# mgmt_providertemplates should look like:
# {
# t1: [prov1, prov2],
# t2: [prov1],
# t3: [prov2]
# }
mgmt_providertemplates = defaultdict(list)
# filter out any empty results from pulling mgmt_templates
for prov_templates in [mt for mt in mgmt_templates if mt is not None]:
# expecting one key (provider), one value (list of templates)
for prov_key, templates in prov_templates.items():
for template in templates:
mgmt_providertemplates[template].append(prov_key)
logger.debug('DEBUG: template_providers: %r', mgmt_providertemplates)
logger.debug('DEBUG: working_providers: %r', working_providers)
usable = {'usable': mark_usable} if mark_usable is not None else {}
# init these outside conditions/looping to be safe in reporting
ignored_providertemplates = defaultdict(list)
tb_pts_to_add = list()
tb_pts_to_delete = list()
tb_templates_to_delete = list()
# ADD PROVIDERTEMPLATES
# add all parseable providertemplates from what is actually on providers
for template_name, provider_keys in mgmt_providertemplates.items():
# drop empty names, or sprout groups
# go over templates pulled from provider mgmt interfaces,
if template_name.strip() == '':
logger.info('Ignoring empty name template on providers %s',
provider_keys)
template_info = TemplateName.parse_template(template_name)
template_group = template_info.group_name
# Don't want sprout templates, or templates that aren't parsable cfme/MIQ
if template_group in GROUPS_TO_IGNORE:
ignored_providertemplates[template_group].append(template_name)
continue
tb_pts_to_add = [
(
template_group,
provider_key,
template_name,
None, # custom_data
usable) for provider_key in provider_keys
]
logger.info(
'Threading add providertemplate records to trackerbot for %s',
template_name)
with ThreadPool(8) as pool:
# thread for each template, passing the list of providers with the template
add_results = pool.starmap(trackerbot.add_provider_template,
tb_pts_to_add)
if not all(
[True if result in [None, True] else False for result in add_results]):
# ignore results that are None, warn for any false results from adding
logger.warning('Trackerbot providertemplate add failed, see logs')
for group, names in ignored_providertemplates.items():
logger.info('Skipped group [%s] templates %r', group, names)
# REMOVE PROVIDERTEMPLATES
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
logger.info(
'Querying providertemplate records from Trackerbot for ones to delete')
pts = trackerbot.depaginate(
tb_api,
tb_api.providertemplate.get(provider_in=working_providers))['objects']
for pt in pts:
key = pt['provider']['key']
pt_name, pt_group = pt['template']['name'], pt['template']['group'][
'name']
if pt_group in GROUPS_TO_IGNORE or key not in mgmt_providertemplates[
pt_name]:
logger.info(
"Marking trackerbot providertemplate for delete: %s::%s", key,
pt_name)
tb_pts_to_delete.append(ProvTemplate(key, pt_name))
with ThreadPool(8) as pool:
# thread for each delete_provider_template call
pool.starmap(trackerbot.delete_provider_template,
((tb_api, pt.provider_key, pt.template_name)
for pt in tb_pts_to_delete))
# REMOVE TEMPLATES
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(tb_api,
tb_api.template.get())['objects']:
template_name = template['name']
if not template['providers'] and template_name.strip():
logger.info("Deleting trackerbot template %s (no providers)",
template_name)
tb_templates_to_delete.append(template_name)
tb_api.template(template_name).delete()
# WRITE REPORT
with open(kwargs.get('outfile'), 'a') as report:
add_header = '##### ProviderTemplate records added: #####\n'
del_header = '##### ProviderTemplate records deleted: #####\n'
report.write(add_header)
add_message = tabulate(sorted([(ptadd[0], ptadd[1], ptadd[2])
for ptadd in tb_pts_to_add],
key=lambda ptadd: ptadd[0]),
headers=['Group', 'Provider', 'Template'],
tablefmt='orgtbl')
report.write(f'{add_message}\n\n')
report.write(del_header)
del_message = tabulate(sorted(
[(ptdel.provider_key, ptdel.template_name)
for ptdel in tb_pts_to_delete],
key=lambda ptdel: ptdel[0]),
headers=['Provider', 'Template'],
tablefmt='orgtbl')
report.write(del_message)
logger.info('%s %s', add_header, add_message)
logger.info('%s %s', del_header, del_message)
return 0
sys.exit(1)
if cmd_args.stream_url:
streams = {stream: cmd_args.stream_url}
elif stream:
streams = cmd_args.stream
else:
streams = ALL_STREAMS
thread_queue = []
for stream in streams:
stream_url = ALL_STREAMS.get(stream)
image_url = cmd_args.image_url
if not cmd_args.template_name:
template_name = TemplateName(stream_url).template_name
else:
template_name = "{}-{}".format(cmd_args.template_name, stream)
if cmd_args.print_name_only:
logger.info("%s Template name: %s", stream, template_name)
continue
for provider_type in provider_types:
providers = list_provider_keys(provider_type)
if cmd_args.provider:
providers = filter(lambda x: x in providers, cmd_args.provider)
for provider in providers:
if provider not in list_provider_keys(provider_type):
def poke_trackerbot(self):
"""This beat-scheduled task periodically polls the trackerbot if there are any new templates.
"""
template_usability = []
# Extract data from trackerbot
tbapi = trackerbot()
objects = depaginate(
tbapi,
tbapi.providertemplate().get(limit=TRACKERBOT_PAGINATE))["objects"]
per_group = {}
for obj in objects:
if obj["template"]["group"]["name"] == 'unknown':
continue
if obj["template"]["group"]["name"] not in per_group:
per_group[obj["template"]["group"]["name"]] = []
per_group[obj["template"]["group"]["name"]].append(obj)
# Sort them using the build date
for group in list(per_group.keys()):
per_group[group] = sorted(per_group[group],
reverse=True,
key=lambda o: o["template"]["datestamp"])
objects = []
# And interleave the the groups
while any(per_group.values()):
for key in list(per_group.keys()):
if per_group[key]:
objects.append(per_group[key].pop(0))
for template in objects:
if template["provider"]["key"] not in list(
conf.cfme_data.management_systems.keys()):
# If we don't use that provider in yamls, set the template as not usable
# 1) It will prevent adding this template if not added
# 2) It'll mark the template as unusable if it already exists
template["usable"] = False
template_usability.append(
(template["provider"]["key"], template["template"]["name"],
template["usable"]))
if not template["usable"]:
continue
group, create = Group.objects.get_or_create(
id=template["template"]["group"]["name"])
# Check if the template is already obsolete
if group.template_obsolete_days is not None:
build_date = parsetime.from_iso_date(
template["template"]["datestamp"])
if build_date <= (parsetime.today() -
timedelta(days=group.template_obsolete_days)):
# It is already obsolete, so ignore it
continue
if conf.cfme_data.management_systems.get(
template["provider"]["key"],
{}).get(
"use_for_sprout", False
): # only create provider in db if it is marked to use for sprout
provider, create = Provider.objects.get_or_create(
id=template["provider"]["key"])
else:
continue
if not provider.is_working:
continue
if "sprout" not in provider.provider_data:
continue
if not provider.provider_type:
provider.provider_type = provider.provider_data.get('type')
provider.save(update_fields=['provider_type'])
template_name = template["template"]["name"]
ga_released = template['template']['ga_released']
custom_data = template['template'].get('custom_data', "{}")
processed_custom_data = yaml.safe_load(custom_data)
template_info = TemplateName.parse_template(template_name)
if not template_info.datestamp:
# Not a CFME/MIQ template, ignore it.
continue
# Original one
original_template = None
try:
original_template = Template.objects.get(
provider=provider,
template_group=group,
original_name=template_name,
name=template_name,
preconfigured=False)
if original_template.ga_released != ga_released:
original_template.ga_released = ga_released
original_template.save(update_fields=['ga_released'])
if provider.provider_type == 'openshift':
if original_template.custom_data != custom_data:
original_template.custom_data = processed_custom_data
original_template.template_type = Template.OPENSHIFT_POD
original_template.container = 'cloudforms-0'
original_template.save(update_fields=[
'custom_data', 'container', 'template_type'
])
except ObjectDoesNotExist:
if template_name in provider.templates:
if template_info.datestamp is None:
self.logger.warning(
"Ignoring template {} because it does not have a date!"
.format(template_name))
continue
template_version = template_info.version
if template_version is None:
# Make up a faux version
# First 3 fields of version get parsed as a zstream
# therefore ... makes it a "nil" stream
template_version = "...{}".format(
template_info.datestamp.strftime("%Y%m%d"))
# openshift has two templates bound to one build version
# sometimes sprout tries using second template -extdb that's wrong
# so, such template has to be marked as not usable inside sprout
usable = not template_name.endswith('-extdb')
with transaction.atomic():
tpl = Template(provider=provider,
template_group=group,
original_name=template_name,
name=template_name,
preconfigured=False,
date=template_info.datestamp,
ready=True,
exists=True,
usable=usable,
version=template_version)
tpl.save()
if provider.provider_type == 'openshift':
tpl.custom_data = processed_custom_data
tpl.container = 'cloudforms-0'
tpl.template_type = Template.OPENSHIFT_POD
tpl.save(update_fields=[
'container', 'template_type', 'custom_data'
])
original_template = tpl
self.logger.info("Created a new template #{}".format(
tpl.id))
# If the provider is set to not preconfigure templates, do not bother even doing it.
if provider.num_simultaneous_configuring > 0:
# Preconfigured one
try:
# openshift providers don't have preconfigured templates.
# so regular template should be used
if provider.provider_type != 'openshift':
preconfigured_template = Template.objects.get(
provider=provider,
template_group=group,
original_name=template_name,
preconfigured=True)
else:
preconfigured_template = Template.objects.get(
provider=provider,
template_group=group,
name=template_name,
preconfigured=True)
preconfigured_template.custom_data = processed_custom_data
preconfigured_template.container = 'cloudforms-0'
preconfigured_template.template_type = Template.OPENSHIFT_POD
preconfigured_template.save(update_fields=[
'container', 'template_type', 'custom_data'
])
if preconfigured_template.ga_released != ga_released:
preconfigured_template.ga_released = ga_released
preconfigured_template.save(update_fields=['ga_released'])
except ObjectDoesNotExist:
if template_name in provider.templates and provider.provider_type != 'openshift':
original_id = original_template.id if original_template is not None else None
create_appliance_template.delay(
provider.id,
group.id,
template_name,
source_template_id=original_id)
# If any of the templates becomes unusable, let sprout know about it
# Similarly if some of them becomes usable ...
for provider_id, template_name, usability in template_usability:
if conf.cfme_data.management_systems.get(provider_id, {}).get(
"use_for_sprout", False
): # only create provider in db if it is marked to use for sprout
provider, create = Provider.objects.get_or_create(id=provider_id)
else:
continue
if not provider.working or provider.disabled:
continue
with transaction.atomic():
for template in Template.objects.filter(
provider=provider, original_name=template_name):
template.usable = usability
template.save(update_fields=['usable'])
# Kill all shepherd appliances if they were accidentally spun up
if not usability:
for appliance in Appliance.objects.filter(
template=template,
marked_for_deletion=False,
appliance_pool=None):
self.logger.info('Killing an appliance {}/{} '
'because its template was marked '
'as unusable'.format(
appliance.id, appliance.name))
Appliance.kill(appliance)
def create_appliance_template(self,
provider_id,
group_id,
template_name,
source_template_id=None):
"""This task creates a template from a fresh CFME template. In case of fatal error during the
operation, the template object is deleted to make sure the operation will be retried next time
when poke_trackerbot runs."""
provider = Provider.objects.get(id=provider_id,
working=True,
disabled=False)
provider.cleanup() # Precaution
group = Group.objects.get(id=group_id)
with transaction.atomic():
# Limit the number of concurrent template configurations
if provider.remaining_configuring_slots == 0:
return False # It will be kicked again when trackerbot gets poked
try:
Template.objects.get(template_group=group,
provider=provider,
original_name=template_name,
preconfigured=True)
return False
except ObjectDoesNotExist:
pass
# Fire off the template preparation
template_info = TemplateName.parse_template(template_name)
template_date_fmt = template_info.datestamp.strftime("%Y%m%d")
if not template_info.datestamp:
return
# Make up a faux version
# First 3 fields of version get parsed as a zstream
# therefore ... makes it a "nil" stream
template_version = template_info.version or "...{}".format(
template_date_fmt)
new_template_name = settings.TEMPLATE_FORMAT.format(
group=group.id,
date=template_date_fmt,
rnd=fauxfactory.gen_alphanumeric(8))
if provider.template_name_length is not None:
allowed_length = provider.template_name_length
# There is some limit
if len(new_template_name) > allowed_length:
# Cut it down
randoms_length = len(new_template_name.rsplit("_", 1)[-1])
minimum_length = (len(new_template_name) -
randoms_length) + 1 # One random must be
if minimum_length <= allowed_length:
# Just cut it
new_template_name = new_template_name[:allowed_length]
else:
# Another solution
new_template_name = settings.TEMPLATE_FORMAT.format(
group=group.id[:2],
date=template_date_fmt, # Use only first 2 of grp
rnd=fauxfactory.gen_alphanumeric(
2)) # And just 2 chars random
# TODO: If anything larger comes, do fix that!
if source_template_id is not None:
try:
source_template = Template.objects.get(id=source_template_id)
except ObjectDoesNotExist:
source_template = None
else:
source_template = None
template = Template(provider=provider,
template_group=group,
name=new_template_name,
date=template_info.datestamp,
version=template_version,
original_name=template_name,
parent_template=source_template,
exists=False)
template.save()
workflow = chain(
prepare_template_deploy.si(template.id),
prepare_template_verify_version.si(template.id),
prepare_template_configure.si(template.id),
prepare_template_seal.si(template.id),
prepare_template_poweroff.si(template.id),
prepare_template_finish.si(template.id),
)
workflow.link_error(prepare_template_delete_on_error.si(template.id))
workflow()
def main(trackerbot_url, mark_usable=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = set(list_provider_keys())
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
ipaddress = cfme_data['management_systems'][provider_key].get(
'ipaddress')
if ipaddress and not net.is_pingable(ipaddress):
continue
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers,
unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [
pt['id'] for pt in trackerbot.depaginate(
api, api.providertemplate.get())['objects']
]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
group_name, datestamp, stream, version = TemplateName.parse_template(
template_name)
# Don't want sprout templates
if group_name in ('sprout', 'rhevm-internal'):
print('Ignoring {} from group {}'.format(template_name,
group_name))
continue
seen_templates.add(template_name)
group = trackerbot.Group(group_name, stream=stream)
template = trackerbot.Template(template_name, group, datestamp)
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name,
provider_key) in existing_provider_templates:
print('Template {} already tracked for provider {}'.format(
template_name, provider_key))
continue
try:
trackerbot.mark_provider_template(api, provider, template,
**usable)
print('Added {} template {} on provider {} (datestamp: {})'.
format(group_name, template_name, provider_key,
datestamp))
except SlumberHttpBaseException as ex:
print("{}\t{}".format(ex.response.status_code, ex.content))
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in trackerbot.depaginate(api,
api.providertemplate.get())['objects']:
provider_key, template_name = pt['provider']['key'], pt['template'][
'name']
if provider_key not in template_providers[template_name] \
and provider_key not in unresponsive_providers:
if provider_key in all_providers:
print("Cleaning up template {} on {}".format(
template_name, provider_key))
trackerbot.delete_provider_template(api, provider_key,
template_name)
else:
print("Skipping template cleanup {} on unknown provider {}".
format(template_name, provider_key))
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(api, api.template.get())['objects']:
if not template['providers']:
print("Deleting template {} (no providers)".format(
template['name']))
api.template(template['name']).delete()
def test_template_parsing(tmp_name, ver_string):
assert TemplateName.parse_template(tmp_name).version == ver_string
def main(trackerbot_url, mark_usable=None, selected_provider=None, **kwargs):
tb_api = trackerbot.api(trackerbot_url)
all_providers = set(
selected_provider
or [key for key, data in cfme_data.management_systems.items()
if 'disabled' not in data.get('tags', [])])
bad_providers = manager.Queue()
# starmap the list of provider_keys into templates_on_provider
# return is list of ProvTemplate tuples
with ThreadPool(8) as pool:
mgmt_templates = pool.starmap(
templates_on_provider,
((provider_key, bad_providers) for provider_key in all_providers)
)
# filter out the misbehaving providers
bad_provider_keys = []
while not bad_providers.empty():
bad_provider_keys.append(bad_providers.get())
logger.warning('Filtering out providers that failed template query: %s', bad_provider_keys)
working_providers = set([key for key in all_providers if key not in bad_provider_keys])
# Flip mgmt_templates into dict keyed on template name, listing providers
# [
# {prov1: [t1, t2]},
# {prov2: [t1, t3]},
# ]
#
# mgmt_providertemplates should look like:
# {
# t1: [prov1, prov2],
# t2: [prov1],
# t3: [prov2]
# }
mgmt_providertemplates = defaultdict(list)
# filter out any empty results from pulling mgmt_templates
for prov_templates in [mt for mt in mgmt_templates if mt is not None]:
# expecting one key (provider), one value (list of templates)
for prov_key, templates in prov_templates.items():
for template in templates:
mgmt_providertemplates[template].append(prov_key)
logger.debug('DEBUG: template_providers: %r', mgmt_providertemplates)
logger.debug('DEBUG: working_providers: %r', working_providers)
usable = {'usable': mark_usable} if mark_usable is not None else {}
# init these outside conditions/looping to be safe in reporting
ignored_providertemplates = defaultdict(list)
tb_pts_to_add = list()
tb_pts_to_delete = list()
tb_templates_to_delete = list()
# ADD PROVIDERTEMPLATES
# add all parseable providertemplates from what is actually on providers
for template_name, provider_keys in mgmt_providertemplates.items():
# drop empty names, or sprout groups
# go over templates pulled from provider mgmt interfaces,
if template_name.strip() == '':
logger.info('Ignoring empty name template on providers %s', provider_keys)
template_info = TemplateName.parse_template(template_name)
template_group = template_info.group_name
# Don't want sprout templates, or templates that aren't parsable cfme/MIQ
if template_group in GROUPS_TO_IGNORE:
ignored_providertemplates[template_group].append(template_name)
continue
tb_pts_to_add = [
(template_group,
provider_key,
template_name,
None, # custom_data
usable)
for provider_key in provider_keys
]
logger.info('Threading add providertemplate records to trackerbot for %s', template_name)
with ThreadPool(8) as pool:
# thread for each template, passing the list of providers with the template
add_results = pool.starmap(
trackerbot.add_provider_template,
tb_pts_to_add
)
if not all([True if result in [None, True] else False for result in add_results]):
# ignore results that are None, warn for any false results from adding
logger.warning('Trackerbot providertemplate add failed, see logs')
for group, names in ignored_providertemplates.items():
logger.info('Skipped group [%s] templates %r', group, names)
# REMOVE PROVIDERTEMPLATES
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
logger.info('Querying providertemplate records from Trackerbot for ones to delete')
pts = trackerbot.depaginate(
tb_api,
tb_api.providertemplate.get(provider_in=working_providers)
)['objects']
for pt in pts:
key = pt['provider']['key']
pt_name, pt_group = pt['template']['name'], pt['template']['group']['name']
if pt_group in GROUPS_TO_IGNORE or key not in mgmt_providertemplates[pt_name]:
logger.info("Marking trackerbot providertemplate for delete: %s::%s",
key, pt_name)
tb_pts_to_delete.append(ProvTemplate(key, pt_name))
with ThreadPool(8) as pool:
# thread for each delete_provider_template call
pool.starmap(
trackerbot.delete_provider_template,
((tb_api, pt.provider_key, pt.template_name) for pt in tb_pts_to_delete))
# REMOVE TEMPLATES
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(tb_api, tb_api.template.get())['objects']:
template_name = template['name']
if not template['providers'] and template_name.strip():
logger.info("Deleting trackerbot template %s (no providers)", template_name)
tb_templates_to_delete.append(template_name)
tb_api.template(template_name).delete()
# WRITE REPORT
with open(kwargs.get('outfile'), 'a') as report:
add_header = '##### ProviderTemplate records added: #####\n'
del_header = '##### ProviderTemplate records deleted: #####\n'
report.write(add_header)
add_message = tabulate(
sorted([(ptadd[0], ptadd[1], ptadd[2]) for ptadd in tb_pts_to_add],
key=lambda ptadd: ptadd[0]),
headers=['Group', 'Provider', 'Template'],
tablefmt='orgtbl'
)
report.write('{}\n\n'.format(add_message))
report.write(del_header)
del_message = tabulate(
sorted([(ptdel.provider_key, ptdel.template_name) for ptdel in tb_pts_to_delete],
key=lambda ptdel: ptdel[0]),
headers=['Provider', 'Template'],
tablefmt='orgtbl'
)
report.write(del_message)
logger.info('%s %s', add_header, add_message)
logger.info('%s %s', del_header, del_message)
return 0