def run(**kwargs):
"""Calls the functions needed to cleanup templates on RHEVM providers.
This is called either by template_upload_all script, or by main
function.
Args:
**kwargs: Kwargs generated from cfme_data['template_upload']['template_upload_rhevm'].
"""
providers = cfme_data['management_systems']
for provider in [prov for prov in providers if providers[prov]['type'] == 'rhevm']:
# If a provider was passed, only cleanup on it, otherwise all rhevm providers
cli_provider = kwargs.get('provider')
if cli_provider and cli_provider != provider:
continue
print("\n--------Start of {}--------".format(provider))
provider_mgmt = get_mgmt(provider)
try:
# Raise exceptions here to log the end of provider section
if not net.is_pingable(provider_mgmt.kwargs.get('ipaddress')):
raise ValueError('Failed to ping provider.')
elif not is_ovirt_engine_running(provider_mgmt):
print('ovirt-engine service not running..')
raise ValueError('ovirt-engine not running on provider')
print('connecting to provider, to establish api handler')
edomain = kwargs.get('edomain')
if not edomain:
edomain = provider_mgmt.kwargs['template_upload']['edomain']
# Test API connection to provider, raises RequestError
provider_mgmt.api # noqa
except Exception as e:
print('Failed connecting to provider: {}'.format(e))
logger.exception(e)
print("--------FAILURE End of {}--------\n".format(provider))
continue
try:
cleanup_templates(provider_mgmt.api,
edomain,
kwargs.get('days_old'),
kwargs.get('max_templates'))
finally:
change_edomain_state(provider_mgmt,
'maintenance',
edomain)
cleanup_empty_dir_on_edomain(provider_mgmt, edomain)
change_edomain_state(provider_mgmt,
'active',
edomain)
print("--------End of {}--------\n".format(provider))
print("Provider Execution completed")
def run(**kwargs):
thread_queue = []
providers = list_provider_keys("virtualcenter")
if kwargs['provider_data']:
mgmt_sys = providers = kwargs['provider_data']['management_systems']
else:
mgmt_sys = cfme_data.management_systems
# Store thread results, no need to use a lock
# because threads will not be adding new keys
results = {provider: None for provider in providers}
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload') and
mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info('Skipping upload on %s due to block_upload', provider)
continue
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'virtualcenter':
continue
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
else:
creds = credentials[mgmt_sys[provider]['credentials']]
username = creds['username']
password = creds['password']
host_ip = mgmt_sys[provider]['ipaddress']
hostname = mgmt_sys[provider]['hostname']
client = VMWareSystem(hostname, username, password)
if not net.is_pingable(host_ip):
continue
thread = Thread(target=upload_template,
args=(client, hostname, username, password, provider,
kwargs.get('image_url'), kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream'], results))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
failed_providers = [provider for provider, result in results.items() if result is False]
skipped_providers = [provider for provider, result in results.items() if result is None]
passed_providers = [provider for provider, result in results.items() if result]
logger.info("providers skipped: %s", skipped_providers)
logger.info("providers passed: %s", passed_providers)
logger.info("providers failed: %s", failed_providers)
if not passed_providers:
raise Exception("Template upload failed for all providers")
else:
logger.info("Upload passed for at least 1 provider... success!")
def run(**kwargs):
thread_queue = []
providers = list_provider_keys("virtualcenter")
if kwargs['provider_data']:
mgmt_sys = providers = kwargs['provider_data']['management_systems']
else:
mgmt_sys = cfme_data.management_systems
# Store thread results, no need to use a lock
# because threads will not be adding new keys
results = {provider: None for provider in providers}
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload') and
mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info('Skipping upload on %s due to block_upload', provider)
continue
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'virtualcenter':
continue
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
else:
creds = credentials[mgmt_sys[provider]['credentials']]
username = creds['username']
password = creds['password']
host_ip = mgmt_sys[provider]['ipaddress']
hostname = mgmt_sys[provider]['hostname']
client = VMWareSystem(hostname, username, password)
if not net.is_pingable(host_ip):
continue
thread = Thread(target=upload_template,
args=(client, hostname, username, password, provider,
kwargs.get('image_url'), kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream'], results))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
failed_providers = [provider for provider, result in results.items() if result is False]
skipped_providers = [provider for provider, result in results.items() if result is None]
passed_providers = [provider for provider, result in results.items() if result]
logger.info("providers skipped: %s", skipped_providers)
logger.info("providers passed: %s", passed_providers)
logger.info("providers failed: %s", failed_providers)
if not passed_providers:
raise Exception("Template upload failed for all providers")
else:
logger.info("Upload passed for at least 1 provider... success!")
def run(**kwargs):
try:
thread_queue = []
providers = list_provider_keys("openshift")
if kwargs['provider_data']:
mgmt_sys = providers = kwargs['provider_data'][
'management_systems']
else:
mgmt_sys = cfme_data['management_systems']
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload') and
mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info('Skipping upload on {} due to block_upload'.format(
provider))
continue
if 'podtesting' not in mgmt_sys[provider]['tags']:
continue
if kwargs['provider_data']:
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
else:
ssh_creds = credentials[mgmt_sys[provider]['ssh_creds']]
username = ssh_creds['username']
password = ssh_creds['password']
oc_creds = credentials[mgmt_sys[provider]['credentials']]
oc_username = oc_creds['username']
oc_password = oc_creds['password']
host_ip = mgmt_sys[provider]['ipaddress']
hostname = mgmt_sys[provider]['hostname']
upload_parameters = cfme_data['template_upload'][
'template_upload_openshift']
upload_folder = kwargs.get('upload_folder',
upload_parameters['upload_folder'])
if not net.is_pingable(host_ip):
continue
thread = Thread(target=upload_template,
args=(hostname, username, password, provider,
kwargs.get('image_url'),
kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream'],
upload_folder, oc_username, oc_password))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
except Exception:
logger.exception('Exception during run method')
return False
def run(**kwargs):
"""Calls the functions needed to cleanup templates on RHEVM providers.
This is called either by template_upload_all script, or by main
function.
Args:
**kwargs: Kwargs generated from cfme_data['template_upload']['template_upload_rhevm'].
"""
providers = cfme_data['management_systems']
for provider in [
prov for prov in providers if providers[prov]['type'] == 'rhevm'
]:
# If a provider was passed, only cleanup on it, otherwise all rhevm providers
cli_provider = kwargs.get('provider')
if cli_provider and cli_provider != provider:
continue
print("\n--------Start of {}--------".format(provider))
provider_mgmt = get_mgmt(provider)
try:
# Raise exceptions here to log the end of provider section
if not net.is_pingable(provider_mgmt.kwargs.get('ipaddress')):
raise ValueError('Failed to ping provider.')
elif not is_ovirt_engine_running(provider_mgmt):
print('ovirt-engine service not running..')
raise ValueError('ovirt-engine not running on provider')
print('connecting to provider, to establish api handler')
edomain = kwargs.get('edomain')
if not edomain:
edomain = provider_mgmt.kwargs['template_upload']['edomain']
# Test API connection to provider, raises RequestError
provider_mgmt.api # noqa
except Exception as e:
print('Failed connecting to provider: {}'.format(e))
logger.exception(e)
print("--------FAILURE End of {}--------\n".format(provider))
continue
try:
cleanup_templates(provider_mgmt.api, edomain,
kwargs.get('days_old'),
kwargs.get('max_templates'))
finally:
change_edomain_state(provider_mgmt, 'maintenance', edomain)
cleanup_empty_dir_on_edomain(provider_mgmt, edomain)
change_edomain_state(provider_mgmt, 'active', edomain)
print("--------End of {}--------\n".format(provider))
print("Provider Execution completed")
def run(**kwargs):
thread_queue = []
providers = list_provider_keys("openstack")
if kwargs['provider_data']:
provider_data = kwargs['provider_data']
mgmt_sys = providers = provider_data['management_systems']
else:
mgmt_sys = cfme_data.management_systems
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload')
and mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info(
'Skipping upload on {} due to block_upload'.format(provider))
continue
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'openstack':
continue
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
else:
mgmt_sys = cfme_data['management_systems']
rhos_credentials = credentials[mgmt_sys[provider]['credentials']]
default_host_creds = credentials['host_default']
username = rhos_credentials['username']
password = rhos_credentials['password']
sshname = default_host_creds['username']
sshpass = default_host_creds['password']
rhosip = mgmt_sys[provider]['ipaddress']
auth_url = mgmt_sys[provider]['auth_url']
if not net.is_pingable(rhosip):
continue
if not net.net_check(ports.SSH, rhosip):
logger.error("SSH connection to %r:%r failed, port unavailable",
provider, ports.SSH)
continue
thread = Thread(target=upload_template,
args=(rhosip, sshname, sshpass, username, password,
auth_url, provider, kwargs.get('image_url'),
kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream']))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def run(**kwargs):
try:
thread_queue = []
providers = list_provider_keys("openshift")
if kwargs['provider_data']:
mgmt_sys = providers = kwargs['provider_data']['management_systems']
else:
mgmt_sys = cfme_data['management_systems']
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload') and
mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info('Skipping upload on {} due to block_upload'.format(provider))
continue
if 'podtesting' not in mgmt_sys[provider]['tags']:
continue
if kwargs['provider_data']:
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
else:
ssh_creds = credentials[mgmt_sys[provider]['ssh_creds']]
username = ssh_creds['username']
password = ssh_creds['password']
oc_creds = credentials[mgmt_sys[provider]['credentials']]
oc_username = oc_creds['username']
oc_password = oc_creds['password']
host_ip = mgmt_sys[provider]['ipaddress']
hostname = mgmt_sys[provider]['hostname']
upload_parameters = cfme_data['template_upload']['template_upload_openshift']
upload_folder = kwargs.get('upload_folder', upload_parameters['upload_folder'])
if not net.is_pingable(host_ip):
continue
thread = Thread(target=upload_template,
args=(hostname, username, password, provider,
kwargs.get('image_url'), kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream'], upload_folder,
oc_username, oc_password))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
except Exception:
logger.exception('Exception during run method')
return False
def run(**kwargs):
thread_queue = []
providers = list_provider_keys("openstack")
if kwargs['provider_data']:
provider_data = kwargs['provider_data']
mgmt_sys = providers = provider_data['management_systems']
else:
mgmt_sys = cfme_data.management_systems
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload') and
mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info('Skipping upload on {} due to block_upload'.format(provider))
continue
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'openstack':
continue
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
else:
mgmt_sys = cfme_data['management_systems']
rhos_credentials = credentials[mgmt_sys[provider]['credentials']]
default_host_creds = credentials['host_default']
username = rhos_credentials['username']
password = rhos_credentials['password']
sshname = default_host_creds['username']
sshpass = default_host_creds['password']
rhosip = mgmt_sys[provider]['ipaddress']
auth_url = mgmt_sys[provider]['auth_url']
if not net.is_pingable(rhosip):
continue
if not net.net_check(ports.SSH, rhosip):
logger.error("SSH connection to %r:%r failed, port unavailable", provider, ports.SSH)
continue
thread = Thread(target=upload_template,
args=(rhosip, sshname, sshpass, username, password, auth_url, provider,
kwargs.get('image_url'), kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream']))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def run(**kwargs):
try:
thread_queue = []
providers = list_provider_keys("virtualcenter")
if kwargs['provider_data']:
mgmt_sys = providers = kwargs['provider_data'][
'management_systems']
else:
mgmt_sys = cfme_data.management_systems
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload') and
mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info('Skipping upload on {} due to block_upload'.format(
provider))
continue
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'virtualcenter':
continue
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
else:
creds = credentials[mgmt_sys[provider]['credentials']]
username = creds['username']
password = creds['password']
host_ip = mgmt_sys[provider]['ipaddress']
hostname = mgmt_sys[provider]['hostname']
client = VMWareSystem(hostname, username, password)
if not net.is_pingable(host_ip):
continue
thread = Thread(target=upload_template,
args=(client, hostname, username, password,
provider, kwargs.get('image_url'),
kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream']))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
except Exception:
logger.exception('Exception during run method')
return False
def run(**kwargs):
try:
thread_queue = []
providers = list_provider_keys("virtualcenter")
if kwargs['provider_data']:
mgmt_sys = providers = kwargs['provider_data']['management_systems']
else:
mgmt_sys = cfme_data.management_systems
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload') and
mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info('Skipping upload on {} due to block_upload'.format(provider))
continue
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'virtualcenter':
continue
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
else:
creds = credentials[mgmt_sys[provider]['credentials']]
username = creds['username']
password = creds['password']
host_ip = mgmt_sys[provider]['ipaddress']
hostname = mgmt_sys[provider]['hostname']
client = VMWareSystem(hostname, username, password)
if not net.is_pingable(host_ip):
continue
thread = Thread(target=upload_template,
args=(client, hostname, username, password, provider,
kwargs.get('image_url'), kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream']))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
except Exception:
logger.exception('Exception during run method')
return False
def run(**kwargs):
"""Calls all the functions needed to upload new template to RHEVM.
This is called either by template_upload_all script, or by main function.
Args:
**kwargs: Kwargs generated from cfme_data['template_upload']['template_upload_rhevm'].
"""
thread_queue = []
valid_providers = []
providers = list_provider_keys("rhevm")
if kwargs.get('provider_data'):
mgmt_sys = providers = kwargs['provider_data']['management_systems']
for provider in providers:
if kwargs.get('provider_data'):
if mgmt_sys[provider]['type'] != 'rhevm':
continue
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
rhevip = mgmt_sys[provider]['ipaddress']
else:
mgmt_sys = cfme_data['management_systems']
ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds']
sshname = credentials[ssh_rhevm_creds]['username']
sshpass = credentials[ssh_rhevm_creds]['password']
rhevip = mgmt_sys[provider]['ipaddress']
if (mgmt_sys[provider].get('template_upload')
and mgmt_sys[provider]['template_upload'].get('block_upload')):
# Providers template_upload section indicates upload should not happen on this provider
continue
logger.info(
"RHEVM:%r verifying provider's state before template upload",
provider)
if not net.is_pingable(rhevip):
continue
elif not is_ovirt_engine_running(rhevip, sshname, sshpass):
logger.info('RHEVM:%r ovirt-engine service not running..',
provider)
continue
valid_providers.append(provider)
for provider in valid_providers:
if kwargs.get('provider_data'):
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
else:
ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds']
sshname = credentials[ssh_rhevm_creds]['username']
sshpass = credentials[ssh_rhevm_creds]['password']
rhevm_credentials = mgmt_sys[provider]['credentials']
username = credentials[rhevm_credentials]['username']
password = credentials[rhevm_credentials]['password']
rhevip = mgmt_sys[provider]['ipaddress']
thread = Thread(target=upload_template,
args=(rhevip, sshname, sshpass, username, password,
provider, kwargs.get('image_url'),
kwargs.get('template_name'),
kwargs.get('provider_data'), kwargs['stream']))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def run(**kwargs):
"""Calls all the functions needed to upload new template to RHEVM.
This is called either by template_upload_all script, or by main function.
Args:
**kwargs: Kwargs generated from cfme_data['template_upload']['template_upload_rhevm'].
"""
thread_queue = []
valid_providers = []
providers = list_provider_keys("rhevm")
if kwargs['provider_data']:
mgmt_sys = providers = kwargs['provider_data']['management_systems']
for provider in providers:
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'rhevm':
continue
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
rhevip = mgmt_sys[provider]['ipaddress']
else:
mgmt_sys = cfme_data['management_systems']
ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds']
sshname = credentials[ssh_rhevm_creds]['username']
sshpass = credentials[ssh_rhevm_creds]['password']
rhevip = mgmt_sys[provider]['ipaddress']
if (mgmt_sys[provider].get('template_upload') and
mgmt_sys[provider]['template_upload'].get('block_upload')):
# Providers template_upload section indicates upload should not happen on this provider
continue
logger.info("RHEVM:%r verifying provider's state before template upload", provider)
if not net.is_pingable(rhevip):
continue
elif not is_ovirt_engine_running(rhevip, sshname, sshpass):
logger.info('RHEVM:%r ovirt-engine service not running..', provider)
continue
valid_providers.append(provider)
for provider in valid_providers:
if kwargs['provider_data']:
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
else:
ssh_rhevm_creds = mgmt_sys[provider]['ssh_creds']
sshname = credentials[ssh_rhevm_creds]['username']
sshpass = credentials[ssh_rhevm_creds]['password']
rhevm_credentials = mgmt_sys[provider]['credentials']
username = credentials[rhevm_credentials]['username']
password = credentials[rhevm_credentials]['password']
rhevip = mgmt_sys[provider]['ipaddress']
thread = Thread(target=upload_template,
args=(rhevip, sshname, sshpass, username, password, provider,
kwargs.get('image_url'), kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream'],
kwargs['glance']))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def main(trackerbot_url, mark_usable=None, selected_provider=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = (set(list_provider_keys())
if not selected_provider else set(selected_provider))
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
ipaddress = cfme_data.management_systems[provider_key].get('ipaddress')
if ipaddress and not net.is_pingable(ipaddress):
continue
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers,
unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [
pt['id'] for pt in trackerbot.depaginate(
api, api.providertemplate.get())['objects']
]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
template_info = TemplateName.parse_template(template_name)
# Don't want sprout templates
if template_info.group_name in ('sprout', 'rhevm-internal'):
logger.info('Ignoring %s from group %s', template_name,
template_info.group_name)
continue
seen_templates.add(template_name)
group = trackerbot.Group(template_info.group_name,
stream=template_info.stream)
try:
template = trackerbot.Template(template_name, group,
template_info.datestamp)
except ValueError:
logger.exception('Failure parsing provider %s template: %s',
provider_key, template_name)
continue
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name,
provider_key) in existing_provider_templates:
logger.info('Template %s already tracked for provider %s',
template_name, provider_key)
continue
try:
trackerbot.mark_provider_template(api, provider, template,
**usable)
logger.info(
'Added %s template %s on provider %s (datestamp: %s)',
template_info.group_name, template_name, provider_key,
template_info.datestamp)
except SlumberHttpBaseException:
logger.exception('%s: exception marking template %s', provider,
template)
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in trackerbot.depaginate(api,
api.providertemplate.get())['objects']:
key, template_name = pt['provider']['key'], pt['template']['name']
if key not in template_providers[
template_name] and key not in unresponsive_providers:
if key in all_providers:
logger.info("Cleaning up template %s on %s", template_name,
key)
trackerbot.delete_provider_template(api, key, template_name)
else:
logger.info(
"Skipping template cleanup %s on unknown provider %s",
template_name, key)
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(api, api.template.get())['objects']:
if not template['providers']:
logger.info("Deleting template %s (no providers)",
template['name'])
api.template(template['name']).delete()
def main(trackerbot_url, mark_usable=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = set(list_provider_keys())
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
ipaddress = cfme_data['management_systems'][provider_key].get('ipaddress')
if ipaddress and not net.is_pingable(ipaddress):
continue
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers, unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [
pt['id']
for pt
in trackerbot.depaginate(api, api.providertemplate.get())['objects']]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
group_name, datestamp, stream = trackerbot.parse_template(template_name)
# Don't want sprout templates
if group_name in ('sprout', 'rhevm-internal'):
print('Ignoring {} from group {}'.format(template_name, group_name))
continue
seen_templates.add(template_name)
group = trackerbot.Group(group_name, stream=stream)
template = trackerbot.Template(template_name, group, datestamp)
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name, provider_key) in existing_provider_templates:
print('Template {} already tracked for provider {}'.format(
template_name, provider_key))
continue
try:
trackerbot.mark_provider_template(api, provider, template, **usable)
print('Added {} template {} on provider {} (datestamp: {})'.format(
group_name, template_name, provider_key, datestamp))
except SlumberHttpBaseException as ex:
print("{}\t{}".format(ex.response.status_code, ex.content))
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']:
provider_key, template_name = pt['provider']['key'], pt['template']['name']
if provider_key not in template_providers[template_name] \
and provider_key not in unresponsive_providers:
if provider_key in all_providers:
print("Cleaning up template {} on {}".format(template_name, provider_key))
trackerbot.delete_provider_template(api, provider_key, template_name)
else:
print("Skipping template cleanup {} on unknown provider {}".format(
template_name, provider_key))
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(api, api.template.get())['objects']:
if not template['providers']:
print("Deleting template {} (no providers)".format(template['name']))
api.template(template['name']).delete()
def main(trackerbot_url, mark_usable=None, selected_provider=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = (set(list_provider_keys())
if not selected_provider
else set(selected_provider))
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
ipaddress = cfme_data.management_systems[provider_key].get('ipaddress')
if ipaddress and not net.is_pingable(ipaddress):
continue
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers, unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [
pt['id']
for pt
in trackerbot.depaginate(api, api.providertemplate.get())['objects']]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
template_info = TemplateName.parse_template(template_name)
# it turned out that some providers like ec2 may have templates w/o names.
# this is easy protection against such issue.
if not template_name.strip():
logger.warn('Ignoring template w/o name on provider %s', provider_key)
continue
# Don't want sprout templates
if template_info.group_name in ('sprout', 'rhevm-internal'):
logger.info('Ignoring %s from group %s', template_name, template_info.group_name)
continue
seen_templates.add(template_name)
group = trackerbot.Group(template_info.group_name, stream=template_info.stream)
try:
template = trackerbot.Template(template_name, group, template_info.datestamp)
except ValueError:
logger.exception('Failure parsing provider %s template: %s',
provider_key, template_name)
continue
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name, provider_key) in existing_provider_templates:
logger.info('Template %s already tracked for provider %s',
template_name, provider_key)
continue
try:
trackerbot.mark_provider_template(api, provider, template, **usable)
logger.info('Added %s template %s on provider %s (datestamp: %s)',
template_info.group_name,
template_name,
provider_key,
template_info.datestamp)
except SlumberHttpBaseException:
logger.exception('%s: exception marking template %s', provider, template)
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']:
key, template_name = pt['provider']['key'], pt['template']['name']
if key not in template_providers[template_name] and key not in unresponsive_providers:
if key in all_providers:
logger.info("Cleaning up template %s on %s", template_name, key)
trackerbot.delete_provider_template(api, key, template_name)
else:
logger.info("Skipping template cleanup %s on unknown provider %s",
template_name, key)
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(api, api.template.get())['objects']:
if not template['providers'] and template['name'].strip():
logger.info("Deleting template %s (no providers)", template['name'])
api.template(template['name']).delete()
