def main(trackerbot_url, mark_usable=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = set(list_provider_keys())
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
ipaddress = cfme_data['management_systems'][provider_key].get('ipaddress')
if ipaddress and not net.is_pingable(ipaddress):
continue
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers, unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [
pt['id']
for pt
in trackerbot.depaginate(api, api.providertemplate.get())['objects']]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
group_name, datestamp, stream = trackerbot.parse_template(template_name)
# Don't want sprout templates
if group_name in ('sprout', 'rhevm-internal'):
print('Ignoring {} from group {}'.format(template_name, group_name))
continue
seen_templates.add(template_name)
group = trackerbot.Group(group_name, stream=stream)
template = trackerbot.Template(template_name, group, datestamp)
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name, provider_key) in existing_provider_templates:
print('Template {} already tracked for provider {}'.format(
template_name, provider_key))
continue
try:
trackerbot.mark_provider_template(api, provider, template, **usable)
print('Added {} template {} on provider {} (datestamp: {})'.format(
group_name, template_name, provider_key, datestamp))
except SlumberHttpBaseException as ex:
print("{}\t{}".format(ex.response.status_code, ex.content))
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']:
provider_key, template_name = pt['provider']['key'], pt['template']['name']
if provider_key not in template_providers[template_name] \
and provider_key not in unresponsive_providers:
if provider_key in all_providers:
print("Cleaning up template {} on {}".format(template_name, provider_key))
trackerbot.delete_provider_template(api, provider_key, template_name)
else:
print("Skipping template cleanup {} on unknown provider {}".format(
template_name, provider_key))
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(api, api.template.get())['objects']:
if not template['providers']:
print("Deleting template {} (no providers)".format(template['name']))
api.template(template['name']).delete()
def main():
urls = cfme_data['basic_info']['cfme_images_url']
stream = args.stream or cfme_data['template_upload']['stream']
upload_url = args.image_url
provider_type = args.provider_type or cfme_data['template_upload'][
'provider_type']
if args.provider_data is not None:
local_datafile = open(args.provider_data, 'r').read()
create_datafile = open(path.conf_path.strpath + '/provider_data.yaml',
'w')
create_datafile.write(local_datafile)
create_datafile.close()
provider_data = cfme.utils.conf.provider_data
stream = provider_data['stream']
if stream:
urls = {}
image_url = cfme_data['basic_info']['cfme_images_url']
urls[stream] = image_url.get(stream)
if not urls[stream]:
image_url = cfme_data['basic_info']['cfme_old_images_url']
urls[stream] = image_url.get(stream)
if not urls[stream]:
base_url = cfme_data['basic_info']['cfme_old_images_url'][
'base_url']
version = ''.join(re.findall(r'(\d+)', stream))
urls[stream] = \
base_url + '.'.join(version[:2]) + '/' + '.'.join(version) + '/'
for key, url in urls.iteritems():
if stream is not None:
if key != stream:
continue
if upload_url:
# strip trailing slashes just in case
if url.rstrip('/') != upload_url.rstrip('/'):
continue
dir_files = browse_directory(url)
if not dir_files:
continue
checksum_url = url + "SHA256SUM"
try:
urlopen(checksum_url)
except Exception:
logger.exception("No valid checksum file for %r, Skipping", key)
continue
kwargs = {}
module = None
if not provider_type:
sys.exit('specify the provider_type')
if provider_type == 'openstack':
module = 'template_upload_rhos'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'rhevm':
module = 'template_upload_rhevm'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'virtualcenter':
module = 'template_upload_vsphere'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'scvmm':
module = 'template_upload_scvmm'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'gce':
module = 'template_upload_gce'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'ec2':
module = 'template_upload_ec2'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'openshift':
module = 'template_upload_openshift'
if module not in dir_files.iterkeys():
continue
if not module:
logger.error('Could not match module to given provider type')
return 1
kwargs['stream'] = stream
kwargs['image_url'] = dir_files[module]
if args.provider_data is not None:
kwargs['provider_data'] = provider_data
else:
kwargs['provider_data'] = None
if cfme_data['template_upload']['automatic_name_strategy']:
kwargs['template_name'] = template_name(
dir_files[module], dir_files[module + "_date"], checksum_url,
get_version(url))
if not stream:
# Stream is none, using automatic naming strategy, parse stream from template name
template_parser = trackerbot.parse_template(
kwargs['template_name'])
if template_parser.stream:
kwargs['stream'] = template_parser.group_name
logger.info(
"TEMPLATE_UPLOAD_ALL:-----Start of %r upload on: %r--------",
kwargs['template_name'], provider_type)
logger.info("Executing %r with the following kwargs: %r", module,
kwargs)
getattr(__import__(module), "run")(**kwargs)
logger.info(
"TEMPLATE_UPLOAD_ALL:------End of %r upload on: %r--------",
kwargs['template_name'], provider_type)
return 0
def main():
urls = cfme_data['basic_info']['cfme_images_url']
stream = args.stream or cfme_data['template_upload']['stream']
upload_url = args.image_url
provider_type = args.provider_type or cfme_data['template_upload']['provider_type']
if args.provider_data is not None:
local_datafile = open(args.provider_data, 'r').read()
create_datafile = open(path.conf_path.strpath + '/provider_data.yaml', 'w')
create_datafile.write(local_datafile)
create_datafile.close()
provider_data = cfme.utils.conf.provider_data
stream = provider_data['stream']
if stream:
urls = {}
image_url = cfme_data['basic_info']['cfme_images_url']
urls[stream] = image_url.get(stream)
if not urls[stream]:
image_url = cfme_data['basic_info']['cfme_old_images_url']
urls[stream] = image_url.get(stream)
if not urls[stream]:
base_url = cfme_data['basic_info']['cfme_old_images_url']['base_url']
version = ''.join(re.findall(r'(\d+)', stream))
urls[stream] = \
base_url + '.'.join(version[:2]) + '/' + '.'.join(version) + '/'
for key, url in urls.iteritems():
if stream is not None:
if key != stream:
continue
if upload_url:
# strip trailing slashes just in case
if url.rstrip('/') != upload_url.rstrip('/'):
continue
dir_files = browse_directory(url)
if not dir_files:
continue
checksum_url = url + "SHA256SUM"
try:
urlopen(checksum_url)
except Exception:
logger.exception("No valid checksum file for %r, Skipping", key)
continue
kwargs = {}
module = None
if not provider_type:
sys.exit('specify the provider_type')
if provider_type == 'openstack':
module = 'template_upload_rhos'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'rhevm':
module = 'template_upload_rhevm'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'virtualcenter':
module = 'template_upload_vsphere'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'scvmm':
module = 'template_upload_scvmm'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'gce':
module = 'template_upload_gce'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'ec2':
module = 'template_upload_ec2'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'openshift':
module = 'template_upload_openshift'
if module not in dir_files.iterkeys():
continue
if not module:
logger.error('Could not match module to given provider type')
return 1
kwargs['stream'] = stream
kwargs['image_url'] = dir_files[module]
if args.provider_data is not None:
kwargs['provider_data'] = provider_data
else:
kwargs['provider_data'] = None
if cfme_data['template_upload']['automatic_name_strategy']:
kwargs['template_name'] = template_name(
dir_files[module],
dir_files[module + "_date"],
checksum_url,
get_version(url)
)
if not stream:
# Stream is none, using automatic naming strategy, parse stream from template name
template_parser = trackerbot.parse_template(kwargs['template_name'])
if template_parser.stream:
kwargs['stream'] = template_parser.group_name
logger.info("TEMPLATE_UPLOAD_ALL:-----Start of %r upload on: %r--------",
kwargs['template_name'], provider_type)
logger.info("Executing %r with the following kwargs: %r", module, kwargs)
getattr(__import__(module), "run")(**kwargs)
logger.info("TEMPLATE_UPLOAD_ALL:------End of %r upload on: %r--------",
kwargs['template_name'], provider_type)
return 0