def pytest_configure(config):
if store.parallelizer_role == 'master' or trackerbot.conf.get('url') is None:
return
# A further optimization here is to make the calls to trackerbot per provider
# and perhaps only pull the providers that are needed, however that will need
# to ensure that the tests that just randomly use providers adhere to the filters
# which may be too tricky right now.
count = 0
if not config.getoption('use_template_cache'):
write_line("Loading templates from trackerbot...")
provider_templates = trackerbot.provider_templates(trackerbot.api())
for provider in list_provider_keys():
TEMPLATES[provider] = provider_templates.get(provider, [])
config.cache.set('miq-trackerbot/{}'.format(provider), TEMPLATES[provider])
count += len(TEMPLATES[provider])
else:
write_line("Using templates from cache...")
provider_templates = None
for provider in list_provider_keys():
templates = config.cache.get('miq-trackerbot/{}'.format(provider), None)
if templates is None:
write_line("Loading templates for {} from source as not in cache".format(provider))
if not provider_templates:
provider_templates = trackerbot.provider_templates(trackerbot.api())
templates = provider_templates.get(provider, [])
config.cache.set('miq-trackerbot/{}'.format(provider), templates)
count += len(templates)
TEMPLATES[provider] = templates
write_line(" Loaded {} templates successfully!".format(count))
def pytest_configure(config):
if store.parallelizer_role == 'master' or trackerbot.conf.get('url') is None:
return
# A further optimization here is to make the calls to trackerbot per provider
# and perhaps only pull the providers that are needed, however that will need
# to ensure that the tests that just randomly use providers adhere to the filters
# which may be too tricky right now.
count = 0
if not config.getoption('use_template_cache'):
store.terminalreporter.line("Loading templates from trackerbot...", green=True)
provider_templates = trackerbot.provider_templates(trackerbot.api())
for provider in list_provider_keys():
TEMPLATES[provider] = provider_templates.get(provider, [])
config.cache.set('miq-trackerbot/{}'.format(provider), TEMPLATES[provider])
count += len(TEMPLATES[provider])
else:
store.terminalreporter.line("Using templates from cache...", green=True)
provider_templates = None
for provider in list_provider_keys():
templates = config.cache.get('miq-trackerbot/{}'.format(provider), None)
if templates is None:
store.terminalreporter.line(
"Loading templates for {} from source as not in cache".format(
provider), green=True)
if not provider_templates:
provider_templates = trackerbot.provider_templates(trackerbot.api())
templates = provider_templates.get(provider, [])
config.cache.set('miq-trackerbot/{}'.format(provider), templates)
count += len(templates)
TEMPLATES[provider] = templates
store.terminalreporter.line(" Loaded {} templates successfully!".format(count), green=True)
def pytest_configure():
if store.parallelizer_role == 'master' or 'url' not in trackerbot.conf:
return
write_line("Loading templates from trackerbot")
provider_templates = trackerbot.provider_templates(trackerbot.api())
for provider in filtered.providers:
TEMPLATES[provider] = provider_templates.get(provider, [])
def build_numbers(self):
api = trackerbot.api()
builds = trackerbot.depaginate(
api, api.build.get(job_name=self.job_name, template=self.template))
build_numbers = []
# XXX relying on trackerbot giving us the most recent builds first, should be explicit
for build in builds.get('objects', []):
if (build['number'] not in self.exclude_builds
and build['number'] >= self.minimum_build):
build_numbers.append(build['number'])
if self.num_builds and len(build_numbers) == self.num_builds:
break
if build_numbers:
print('Pulling reports from builds {}'.format(', '.join(
[str(n) for n in build_numbers])))
return build_numbers
def build_numbers(self):
api = trackerbot.api()
builds = trackerbot.depaginate(api,
api.build.get(job_name=self.job_name, template=self.template)
)
build_numbers = []
# XXX relying on trackerbot giving us the most recent builds first, should be explicit
for build in builds.get('objects', []):
if (build['number'] not in self.exclude_builds and
build['number'] >= self.minimum_build):
build_numbers.append(build['number'])
if self.num_builds and len(build_numbers) == self.num_builds:
break
if build_numbers:
print('Pulling reports from builds {}'.format(
', '.join([str(n) for n in build_numbers])))
return build_numbers
def main(trackerbot_url, mark_usable=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
# Queue up list_template calls
for provider_key in list_all_providers():
if provider_key.startswith('ec2') or provider_key.startswith('rhevm'):
continue
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
# Find some templates and update the API
for template_name, providers in template_providers.items():
try:
stream, datestamp = trackerbot.parse_template(template_name)
except ValueError:
# No matches
continue
group = trackerbot.Group(stream)
template = trackerbot.Template(template_name, group, datestamp)
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
try:
trackerbot.mark_provider_template(api, provider, template,
usable=mark_usable, tested=False)
print 'template %s updated -- %s %s %r, marked usable: %s' % (
template, stream, datestamp, providers, bool(mark_usable))
except SlumberHttpBaseException as ex:
print ex.response.status_code, ex.content
def main(trackerbot_url, mark_usable=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = set(list_all_providers())
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers, unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [
pt['id']
for pt
in trackerbot.depaginate(api, api.providertemplate.get())['objects']]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
group_name, datestamp, stream = trackerbot.parse_template(template_name)
# Don't want sprout templates
if group_name in ('sprout', 'rhevm-internal'):
print 'Ignoring %s from group %s' % (template_name, group_name)
continue
seen_templates.add(template_name)
group = trackerbot.Group(group_name, stream=stream)
template = trackerbot.Template(template_name, group, datestamp)
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name, provider_key) in existing_provider_templates:
print 'Template %s already tracked for provider %s' % (template_name, provider_key)
continue
try:
trackerbot.mark_provider_template(api, provider, template, **usable)
print 'Added %s template %s on provider %s (datestamp: %s)' % (
group_name, template_name, provider_key, datestamp)
except SlumberHttpBaseException as ex:
print ex.response.status_code, ex.content
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']:
provider_key, template_name = pt['provider']['key'], pt['template']['name']
if provider_key not in template_providers[template_name] \
and provider_key not in unresponsive_providers:
if provider_key in all_providers:
print "Cleaning up template %s on %s" % (template_name, provider_key)
trackerbot.delete_provider_template(api, provider_key, template_name)
else:
print "Skipping template cleanup %s on unknown provider %s" % (
template_name, provider_key)
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(api, api.template.get())['objects']:
if not template['providers']:
print "Deleting template %s (no providers)" % template['name']
api.template(template['name']).delete()
stream_data['passed_on_providers'])
elif filter(lambda x: 'scvmm' in x, stream_data['failed_on_providers']):
report.write('\n\nFAILED: {}'.format(
images_uploaded(stream)['template_scvmm']))
map(lambda(x): report.write(
'\n{}: Failed'.format(x)) if 'scvmm' in x else '',
stream_data['failed_on_providers'])
else:
print('\n\nMISSING: SCVMM template is not available on any '
'scvmm providers yet')
report.write('\n\nMISSING: SCVMM template is not available on any '
'scvmm providers yet')
report.seek(0, 0)
if "FAILED" in report.read():
status = "FAILED"
report.close()
update_template_log(appliance_template, action='remove')
print("template_tester_results report generated:{}".format(status))
else:
print("No Templates tested on: {}".format(datetime.datetime.now()))
if __name__ == '__main__':
args = parse_cmd_line()
api = trackerbot.api(args.trackerbot_url)
if not args.stream or not args.appliance_template:
sys.exit("stream and appliance_template "
"cannot be None, specify the stream as --stream <stream-name>"
"and template as --template <template-name>")
generate_html_report(api, args.stream, args.output, args.appliance_template, args.provider)
def main(trackerbot_url, mark_usable=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = set(list_all_providers())
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers, unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [pt['id'] for pt in api.providertemplate.get(limit=0)['objects']]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
group_name, datestamp, stream = trackerbot.parse_template(template_name)
# Don't want sprout templates
if group_name in ('sprout', 'rhevm-internal'):
print 'Ignoring %s from group %s' % (template_name, group_name)
continue
seen_templates.add(template_name)
group = trackerbot.Group(group_name, stream=stream)
template = trackerbot.Template(template_name, group, datestamp)
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name, provider_key) in existing_provider_templates:
print 'Template %s already tracked for provider %s' % (template_name, provider_key)
continue
try:
trackerbot.mark_provider_template(api, provider, template, **usable)
print 'Added %s template %s on provider %s (datestamp: %s)' % (
group_name, template_name, provider_key, datestamp)
except SlumberHttpBaseException as ex:
print ex.response.status_code, ex.content
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in api.providertemplate.get(limit=0)['objects']:
provider_key, template_name = pt['provider']['key'], pt['template']['name']
if provider_key not in template_providers[template_name] \
and provider_key not in unresponsive_providers:
if provider_key in all_providers:
print "Cleaning up template %s on %s" % (template_name, provider_key)
trackerbot.delete_provider_template(api, provider_key, template_name)
else:
print "Skipping template cleanup %s on unknown provider %s" % (
template_name, provider_key)
# Remove templates that aren't on any providers anymore
for template in api.template.get(limit=0)['objects']:
if not template['providers']:
print "Deleting template %s (no providers)" % template['name']
api.template(template['name']).delete()
import fauxfactory
import traceback
import dockerbot
import json
import requests
from utils.conf import docker as docker_conf
from utils.appliance import Appliance
from utils.trackerbot import api
from utils.log import create_logger
from slumber.exceptions import HttpClientError
token = docker_conf['gh_token']
owner = docker_conf['gh_owner']
repo = docker_conf['gh_repo']
tapi = api()
CONT_LIMIT = docker_conf['workers']
DEBUG = docker_conf.get('debug', False)
logger = create_logger('check_prs', 'prt.log')
def perform_request(url):
""" Simple function to assist in performing GET requests from github
Runs if there is a token, else just spits out a blank dict
Args:
url: The url to process, which is anything after the "...repos/"
import fauxfactory
import traceback
import dockerbot
import json
import requests
from utils.conf import docker as docker_conf
from utils.appliance import Appliance
from utils.trackerbot import api
from utils.log import create_logger
from slumber.exceptions import HttpClientError
token = docker_conf['gh_token']
owner = docker_conf['gh_owner']
repo = docker_conf['gh_repo']
tapi = api()
CONT_LIMIT = docker_conf['workers']
DEBUG = docker_conf.get('debug', False)
logger = create_logger('check_prs', 'prt.log')
def perform_request(url):
""" Simple function to assist in performing GET requests from github
Runs if there is a token, else just spits out a blank dict
Args:
url: The url to process, which is anything after the "...repos/"
def main(trackerbot_url, mark_usable=None):
api = trackerbot.api(trackerbot_url)
thread_q = []
thread_lock = Lock()
template_providers = defaultdict(list)
all_providers = set(list_provider_keys())
unresponsive_providers = set()
# Queue up list_template calls
for provider_key in all_providers:
ipaddress = cfme_data['management_systems'][provider_key].get(
'ipaddress', None)
if ipaddress and not net.is_pingable(ipaddress):
continue
thread = Thread(target=get_provider_templates,
args=(provider_key, template_providers,
unresponsive_providers, thread_lock))
thread_q.append(thread)
thread.start()
# Join the queued calls
for thread in thread_q:
thread.join()
seen_templates = set()
if mark_usable is None:
usable = {}
else:
usable = {'usable': mark_usable}
existing_provider_templates = [
pt['id'] for pt in trackerbot.depaginate(
api, api.providertemplate.get())['objects']
]
# Find some templates and update the API
for template_name, providers in template_providers.items():
template_name = str(template_name)
group_name, datestamp, stream = trackerbot.parse_template(
template_name)
# Don't want sprout templates
if group_name in ('sprout', 'rhevm-internal'):
print('Ignoring {} from group {}'.format(template_name,
group_name))
continue
seen_templates.add(template_name)
group = trackerbot.Group(group_name, stream=stream)
template = trackerbot.Template(template_name, group, datestamp)
for provider_key in providers:
provider = trackerbot.Provider(provider_key)
if '{}_{}'.format(template_name,
provider_key) in existing_provider_templates:
print('Template {} already tracked for provider {}'.format(
template_name, provider_key))
continue
try:
trackerbot.mark_provider_template(api, provider, template,
**usable)
print('Added {} template {} on provider {} (datestamp: {})'.
format(group_name, template_name, provider_key,
datestamp))
except SlumberHttpBaseException as ex:
print("{}\t{}".format(ex.response.status_code, ex.content))
# Remove provider relationships where they no longer exist, skipping unresponsive providers,
# and providers not known to this environment
for pt in trackerbot.depaginate(api,
api.providertemplate.get())['objects']:
provider_key, template_name = pt['provider']['key'], pt['template'][
'name']
if provider_key not in template_providers[template_name] \
and provider_key not in unresponsive_providers:
if provider_key in all_providers:
print("Cleaning up template {} on {}".format(
template_name, provider_key))
trackerbot.delete_provider_template(api, provider_key,
template_name)
else:
print("Skipping template cleanup {} on unknown provider {}".
format(template_name, provider_key))
# Remove templates that aren't on any providers anymore
for template in trackerbot.depaginate(api, api.template.get())['objects']:
if not template['providers']:
print("Deleting template {} (no providers)".format(
template['name']))
api.template(template['name']).delete()
