def main(trackerbot_url, mark_usable=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
# Queue up list_template calls
for provider_key in list_all_providers():
if provider_key.startswith('ec2') or provider_key.startswith('rhevm'):
continue
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
# Find some templates and update the API
for template_name, providers in template_providers.items():
try:
stream, datestamp = trackerbot.parse_template(template_name)
except ValueError:
# No matches
continue
group = trackerbot.Group(stream)
template = trackerbot.Template(template_name, group, datestamp)
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
try:
trackerbot.mark_provider_template(api, provider, template,
usable=mark_usable, tested=False)
print 'template %s updated -- %s %s %r, marked usable: %s' % (
template, stream, datestamp, providers, bool(mark_usable))
except SlumberHttpBaseException as ex:
print ex.response.status_code, ex.content
def main(trackerbot_url, mark_usable=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = set(list_all_providers())
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers, unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [
pt['id']
for pt
in trackerbot.depaginate(api, api.providertemplate.get())['objects']]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
group_name, datestamp, stream = trackerbot.parse_template(template_name)
# Don't want sprout templates
if group_name in ('sprout', 'rhevm-internal'):
print 'Ignoring %s from group %s' % (template_name, group_name)
continue
seen_templates.add(template_name)
group = trackerbot.Group(group_name, stream=stream)
template = trackerbot.Template(template_name, group, datestamp)
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name, provider_key) in existing_provider_templates:
print 'Template %s already tracked for provider %s' % (template_name, provider_key)
continue
try:
trackerbot.mark_provider_template(api, provider, template, **usable)
print 'Added %s template %s on provider %s (datestamp: %s)' % (
group_name, template_name, provider_key, datestamp)
except SlumberHttpBaseException as ex:
print ex.response.status_code, ex.content
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']:
provider_key, template_name = pt['provider']['key'], pt['template']['name']
if provider_key not in template_providers[template_name] \
and provider_key not in unresponsive_providers:
if provider_key in all_providers:
print "Cleaning up template %s on %s" % (template_name, provider_key)
trackerbot.delete_provider_template(api, provider_key, template_name)
else:
print "Skipping template cleanup %s on unknown provider %s" % (
template_name, provider_key)
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(api, api.template.get())['objects']:
if not template['providers']:
print "Deleting template %s (no providers)" % template['name']
api.template(template['name']).delete()
def main(trackerbot_url, mark_usable=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = set(list_all_providers())
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers, unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [pt['id'] for pt in api.providertemplate.get(limit=0)['objects']]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
group_name, datestamp, stream = trackerbot.parse_template(template_name)
# Don't want sprout templates
if group_name in ('sprout', 'rhevm-internal'):
print 'Ignoring %s from group %s' % (template_name, group_name)
continue
seen_templates.add(template_name)
group = trackerbot.Group(group_name, stream=stream)
template = trackerbot.Template(template_name, group, datestamp)
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name, provider_key) in existing_provider_templates:
print 'Template %s already tracked for provider %s' % (template_name, provider_key)
continue
try:
trackerbot.mark_provider_template(api, provider, template, **usable)
print 'Added %s template %s on provider %s (datestamp: %s)' % (
group_name, template_name, provider_key, datestamp)
except SlumberHttpBaseException as ex:
print ex.response.status_code, ex.content
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in api.providertemplate.get(limit=0)['objects']:
provider_key, template_name = pt['provider']['key'], pt['template']['name']
if provider_key not in template_providers[template_name] \
and provider_key not in unresponsive_providers:
if provider_key in all_providers:
print "Cleaning up template %s on %s" % (template_name, provider_key)
trackerbot.delete_provider_template(api, provider_key, template_name)
else:
print "Skipping template cleanup %s on unknown provider %s" % (
template_name, provider_key)
# Remove templates that aren't on any providers anymore
for template in api.template.get(limit=0)['objects']:
if not template['providers']:
print "Deleting template %s (no providers)" % template['name']
api.template(template['name']).delete()
def main():
urls = cfme_data['basic_info']['cfme_images_url']
stream = args.stream or cfme_data['template_upload']['stream']
upload_url = args.image_url
provider_type = args.provider_type or cfme_data['template_upload'][
'provider_type']
if args.provider_data is not None:
local_datafile = open(args.provider_data, 'r').read()
create_datafile = open(path.conf_path.strpath + '/provider_data.yaml',
'w')
create_datafile.write(local_datafile)
create_datafile.close()
provider_data = utils.conf.provider_data
stream = provider_data['stream']
if stream:
urls = {}
image_url = cfme_data['basic_info']['cfme_images_url']
urls[stream] = image_url.get(stream)
if not urls[stream]:
image_url = cfme_data['basic_info']['cfme_old_images_url']
urls[stream] = image_url.get(stream)
if not urls[stream]:
base_url = cfme_data['basic_info']['cfme_old_images_url'][
'base_url']
version = ''.join(re.findall(r'(\d+)', stream))
urls[stream] = \
base_url + '.'.join(version[:2]) + '/' + '.'.join(version) + '/'
for key, url in urls.iteritems():
if stream is not None:
if key != stream:
continue
if upload_url:
# strip trailing slashes just in case
if url.rstrip('/') != upload_url.rstrip('/'):
continue
dir_files = browse_directory(url)
if not dir_files:
continue
checksum_url = url + "SHA256SUM"
try:
urlopen(checksum_url)
except Exception:
print("No valid checksum file for {}. Skipping...".format(key))
continue
kwargs = {}
if not provider_type:
sys.exit('specify the provider_type')
if provider_type == 'openstack':
module = 'template_upload_rhos'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'rhevm':
module = 'template_upload_rhevm'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'virtualcenter':
module = 'template_upload_vsphere'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'scvmm':
module = 'template_upload_scvmm'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'gce':
module = 'template_upload_gce'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'ec2':
module = 'template_upload_ec2'
if module not in dir_files.iterkeys():
continue
kwargs['stream'] = stream
kwargs['image_url'] = dir_files[module]
if args.provider_data is not None:
kwargs['provider_data'] = provider_data
else:
kwargs['provider_data'] = None
if cfme_data['template_upload']['automatic_name_strategy']:
kwargs['template_name'] = template_name(
dir_files[module], dir_files[module + "_date"], checksum_url,
get_version(url))
if not stream:
# Stream is none, using automatic naming strategy, parse stream from template name
template_parser = trackerbot.parse_template(
kwargs['template_name'])
if template_parser.stream:
kwargs['stream'] = template_parser.group_name
print("TEMPLATE_UPLOAD_ALL:-----Start of {} upload on: {}--------".
format(kwargs['template_name'], provider_type))
print("Executing {} with the following kwargs: {}".format(
module, kwargs))
getattr(__import__(module), "run")(**kwargs)
print(
"TEMPLATE_UPLOAD_ALL:------End of {} upload on: {}--------".format(
kwargs['template_name'], provider_type))
def main(trackerbot_url, mark_usable=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = set(list_provider_keys())
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
ipaddress = cfme_data['management_systems'][provider_key].get(
'ipaddress', None)
if ipaddress and not net.is_pingable(ipaddress):
continue
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers,
unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [
pt['id'] for pt in trackerbot.depaginate(
api, api.providertemplate.get())['objects']
]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
group_name, datestamp, stream = trackerbot.parse_template(
template_name)
# Don't want sprout templates
if group_name in ('sprout', 'rhevm-internal'):
print('Ignoring {} from group {}'.format(template_name,
group_name))
continue
seen_templates.add(template_name)
group = trackerbot.Group(group_name, stream=stream)
template = trackerbot.Template(template_name, group, datestamp)
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name,
provider_key) in existing_provider_templates:
print('Template {} already tracked for provider {}'.format(
template_name, provider_key))
continue
try:
trackerbot.mark_provider_template(api, provider, template,
**usable)
print('Added {} template {} on provider {} (datestamp: {})'.
format(group_name, template_name, provider_key,
datestamp))
except SlumberHttpBaseException as ex:
print("{}\t{}".format(ex.response.status_code, ex.content))
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in trackerbot.depaginate(api,
api.providertemplate.get())['objects']:
provider_key, template_name = pt['provider']['key'], pt['template'][
'name']
if provider_key not in template_providers[template_name] \
and provider_key not in unresponsive_providers:
if provider_key in all_providers:
print("Cleaning up template {} on {}".format(
template_name, provider_key))
trackerbot.delete_provider_template(api, provider_key,
template_name)
else:
print("Skipping template cleanup {} on unknown provider {}".
format(template_name, provider_key))
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(api, api.template.get())['objects']:
if not template['providers']:
print("Deleting template {} (no providers)".format(
template['name']))
api.template(template['name']).delete()
def main():
urls = cfme_data['basic_info']['cfme_images_url']
stream = args.stream or cfme_data['template_upload']['stream']
upload_url = args.image_url
provider_type = args.provider_type or cfme_data['template_upload']['provider_type']
if args.provider_data is not None:
local_datafile = open(args.provider_data, 'r').read()
create_datafile = open(path.conf_path.strpath + '/provider_data.yaml', 'w')
create_datafile.write(local_datafile)
create_datafile.close()
provider_data = utils.conf.provider_data
stream = provider_data['stream']
if stream:
urls = {}
image_url = cfme_data['basic_info']['cfme_images_url']
urls[stream] = image_url.get(stream)
if not urls[stream]:
image_url = cfme_data['basic_info']['cfme_old_images_url']
urls[stream] = image_url.get(stream)
if not urls[stream]:
base_url = cfme_data['basic_info']['cfme_old_images_url']['base_url']
version = ''.join(re.findall(r'(\d+)', stream))
urls[stream] = \
base_url + '.'.join(version[:2]) + '/' + '.'.join(version) + '/'
for key, url in urls.iteritems():
if stream is not None:
if key != stream:
continue
if upload_url:
# strip trailing slashes just in case
if url.rstrip('/') != upload_url.rstrip('/'):
continue
dir_files = browse_directory(url)
if not dir_files:
continue
checksum_url = url + "SHA256SUM"
try:
urlopen(checksum_url)
except Exception:
print("No valid checksum file for {}. Skipping...".format(key))
continue
kwargs = {}
if not provider_type:
sys.exit('specify the provider_type')
if provider_type == 'openstack':
module = 'template_upload_rhos'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'rhevm':
module = 'template_upload_rhevm'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'virtualcenter':
module = 'template_upload_vsphere'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'scvmm':
module = 'template_upload_scvmm'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'gce':
module = 'template_upload_gce'
if module not in dir_files.iterkeys():
continue
elif provider_type == 'ec2':
module = 'template_upload_ec2'
if module not in dir_files.iterkeys():
continue
kwargs['stream'] = stream
kwargs['image_url'] = dir_files[module]
if args.provider_data is not None:
kwargs['provider_data'] = provider_data
else:
kwargs['provider_data'] = None
if cfme_data['template_upload']['automatic_name_strategy']:
kwargs['template_name'] = template_name(
dir_files[module],
dir_files[module + "_date"],
checksum_url,
get_version(url)
)
if not stream:
# Stream is none, using automatic naming strategy, parse stream from template name
template_parser = trackerbot.parse_template(kwargs['template_name'])
if template_parser.stream:
kwargs['stream'] = template_parser.group_name
print("TEMPLATE_UPLOAD_ALL:-----Start of {} upload on: {}--------".format(
kwargs['template_name'], provider_type))
print("Executing {} with the following kwargs: {}".format(module, kwargs))
getattr(__import__(module), "run")(**kwargs)
print("TEMPLATE_UPLOAD_ALL:------End of {} upload on: {}--------".format(
kwargs['template_name'], provider_type))