def pytest_configure(config): if store.parallelizer_role == 'master' or trackerbot.conf.get('url') is None: return # A further optimization here is to make the calls to trackerbot per provider # and perhaps only pull the providers that are needed, however that will need # to ensure that the tests that just randomly use providers adhere to the filters # which may be too tricky right now. count = 0 if not config.getoption('use_template_cache'): write_line("Loading templates from trackerbot...") provider_templates = trackerbot.provider_templates(trackerbot.api()) for provider in list_provider_keys(): TEMPLATES[provider] = provider_templates.get(provider, []) config.cache.set('miq-trackerbot/{}'.format(provider), TEMPLATES[provider]) count += len(TEMPLATES[provider]) else: write_line("Using templates from cache...") provider_templates = None for provider in list_provider_keys(): templates = config.cache.get('miq-trackerbot/{}'.format(provider), None) if templates is None: write_line("Loading templates for {} from source as not in cache".format(provider)) if not provider_templates: provider_templates = trackerbot.provider_templates(trackerbot.api()) templates = provider_templates.get(provider, []) config.cache.set('miq-trackerbot/{}'.format(provider), templates) count += len(templates) TEMPLATES[provider] = templates write_line(" Loaded {} templates successfully!".format(count))
def pytest_configure(config): if store.parallelizer_role == 'master' or trackerbot.conf.get('url') is None: return # A further optimization here is to make the calls to trackerbot per provider # and perhaps only pull the providers that are needed, however that will need # to ensure that the tests that just randomly use providers adhere to the filters # which may be too tricky right now. count = 0 if not config.getoption('use_template_cache'): store.terminalreporter.line("Loading templates from trackerbot...", green=True) provider_templates = trackerbot.provider_templates(trackerbot.api()) for provider in list_provider_keys(): TEMPLATES[provider] = provider_templates.get(provider, []) config.cache.set('miq-trackerbot/{}'.format(provider), TEMPLATES[provider]) count += len(TEMPLATES[provider]) else: store.terminalreporter.line("Using templates from cache...", green=True) provider_templates = None for provider in list_provider_keys(): templates = config.cache.get('miq-trackerbot/{}'.format(provider), None) if templates is None: store.terminalreporter.line( "Loading templates for {} from source as not in cache".format( provider), green=True) if not provider_templates: provider_templates = trackerbot.provider_templates(trackerbot.api()) templates = provider_templates.get(provider, []) config.cache.set('miq-trackerbot/{}'.format(provider), templates) count += len(templates) TEMPLATES[provider] = templates store.terminalreporter.line(" Loaded {} templates successfully!".format(count), green=True)
def pytest_configure(): if store.parallelizer_role == 'master' or 'url' not in trackerbot.conf: return write_line("Loading templates from trackerbot") provider_templates = trackerbot.provider_templates(trackerbot.api()) for provider in filtered.providers: TEMPLATES[provider] = provider_templates.get(provider, [])
def build_numbers(self): api = trackerbot.api() builds = trackerbot.depaginate( api, api.build.get(job_name=self.job_name, template=self.template)) build_numbers = [] # XXX relying on trackerbot giving us the most recent builds first, should be explicit for build in builds.get('objects', []): if (build['number'] not in self.exclude_builds and build['number'] >= self.minimum_build): build_numbers.append(build['number']) if self.num_builds and len(build_numbers) == self.num_builds: break if build_numbers: print('Pulling reports from builds {}'.format(', '.join( [str(n) for n in build_numbers]))) return build_numbers
def build_numbers(self): api = trackerbot.api() builds = trackerbot.depaginate(api, api.build.get(job_name=self.job_name, template=self.template) ) build_numbers = [] # XXX relying on trackerbot giving us the most recent builds first, should be explicit for build in builds.get('objects', []): if (build['number'] not in self.exclude_builds and build['number'] >= self.minimum_build): build_numbers.append(build['number']) if self.num_builds and len(build_numbers) == self.num_builds: break if build_numbers: print('Pulling reports from builds {}'.format( ', '.join([str(n) for n in build_numbers]))) return build_numbers
def main(trackerbot_url, mark_usable=None): api = trackerbot.api(trackerbot_url) thread_q = [] thread_lock = Lock() template_providers = defaultdict(list) # Queue up list_template calls for provider_key in list_all_providers(): if provider_key.startswith('ec2') or provider_key.startswith('rhevm'): continue thread = Thread(target=get_provider_templates, args=(provider_key, template_providers, thread_lock)) thread_q.append(thread) thread.start() # Join the queued calls for thread in thread_q: thread.join() # Find some templates and update the API for template_name, providers in template_providers.items(): try: stream, datestamp = trackerbot.parse_template(template_name) except ValueError: # No matches continue group = trackerbot.Group(stream) template = trackerbot.Template(template_name, group, datestamp) for provider_key in providers: provider = trackerbot.Provider(provider_key) try: trackerbot.mark_provider_template(api, provider, template, usable=mark_usable, tested=False) print 'template %s updated -- %s %s %r, marked usable: %s' % ( template, stream, datestamp, providers, bool(mark_usable)) except SlumberHttpBaseException as ex: print ex.response.status_code, ex.content
def main(trackerbot_url, mark_usable=None): api = trackerbot.api(trackerbot_url) thread_q = [] thread_lock = Lock() template_providers = defaultdict(list) all_providers = set(list_all_providers()) unresponsive_providers = set() # Queue up list_template calls for provider_key in all_providers: thread = Thread(target=get_provider_templates, args=(provider_key, template_providers, unresponsive_providers, thread_lock)) thread_q.append(thread) thread.start() # Join the queued calls for thread in thread_q: thread.join() seen_templates = set() if mark_usable is None: usable = {} else: usable = {'usable': mark_usable} existing_provider_templates = [ pt['id'] for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']] # Find some templates and update the API for template_name, providers in template_providers.items(): template_name = str(template_name) group_name, datestamp, stream = trackerbot.parse_template(template_name) # Don't want sprout templates if group_name in ('sprout', 'rhevm-internal'): print 'Ignoring %s from group %s' % (template_name, group_name) continue seen_templates.add(template_name) group = trackerbot.Group(group_name, stream=stream) template = trackerbot.Template(template_name, group, datestamp) for provider_key in providers: provider = trackerbot.Provider(provider_key) if '{}_{}'.format(template_name, provider_key) in existing_provider_templates: print 'Template %s already tracked for provider %s' % (template_name, provider_key) continue try: trackerbot.mark_provider_template(api, provider, template, **usable) print 'Added %s template %s on provider %s (datestamp: %s)' % ( group_name, template_name, provider_key, datestamp) except SlumberHttpBaseException as ex: print ex.response.status_code, ex.content # Remove provider relationships where they no longer exist, skipping unresponsive providers, # and providers not known to this environment for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']: provider_key, template_name = pt['provider']['key'], pt['template']['name'] if provider_key not in template_providers[template_name] \ and provider_key not in unresponsive_providers: if provider_key in all_providers: print "Cleaning up template %s on %s" % (template_name, provider_key) trackerbot.delete_provider_template(api, provider_key, template_name) else: print "Skipping template cleanup %s on unknown provider %s" % ( template_name, provider_key) # Remove templates that aren't on any providers anymore for template in trackerbot.depaginate(api, api.template.get())['objects']: if not template['providers']: print "Deleting template %s (no providers)" % template['name'] api.template(template['name']).delete()
stream_data['passed_on_providers']) elif filter(lambda x: 'scvmm' in x, stream_data['failed_on_providers']): report.write('\n\nFAILED: {}'.format( images_uploaded(stream)['template_scvmm'])) map(lambda(x): report.write( '\n{}: Failed'.format(x)) if 'scvmm' in x else '', stream_data['failed_on_providers']) else: print('\n\nMISSING: SCVMM template is not available on any ' 'scvmm providers yet') report.write('\n\nMISSING: SCVMM template is not available on any ' 'scvmm providers yet') report.seek(0, 0) if "FAILED" in report.read(): status = "FAILED" report.close() update_template_log(appliance_template, action='remove') print("template_tester_results report generated:{}".format(status)) else: print("No Templates tested on: {}".format(datetime.datetime.now())) if __name__ == '__main__': args = parse_cmd_line() api = trackerbot.api(args.trackerbot_url) if not args.stream or not args.appliance_template: sys.exit("stream and appliance_template " "cannot be None, specify the stream as --stream <stream-name>" "and template as --template <template-name>") generate_html_report(api, args.stream, args.output, args.appliance_template, args.provider)
def main(trackerbot_url, mark_usable=None): api = trackerbot.api(trackerbot_url) thread_q = [] thread_lock = Lock() template_providers = defaultdict(list) all_providers = set(list_all_providers()) unresponsive_providers = set() # Queue up list_template calls for provider_key in all_providers: thread = Thread(target=get_provider_templates, args=(provider_key, template_providers, unresponsive_providers, thread_lock)) thread_q.append(thread) thread.start() # Join the queued calls for thread in thread_q: thread.join() seen_templates = set() if mark_usable is None: usable = {} else: usable = {'usable': mark_usable} existing_provider_templates = [pt['id'] for pt in api.providertemplate.get(limit=0)['objects']] # Find some templates and update the API for template_name, providers in template_providers.items(): template_name = str(template_name) group_name, datestamp, stream = trackerbot.parse_template(template_name) # Don't want sprout templates if group_name in ('sprout', 'rhevm-internal'): print 'Ignoring %s from group %s' % (template_name, group_name) continue seen_templates.add(template_name) group = trackerbot.Group(group_name, stream=stream) template = trackerbot.Template(template_name, group, datestamp) for provider_key in providers: provider = trackerbot.Provider(provider_key) if '{}_{}'.format(template_name, provider_key) in existing_provider_templates: print 'Template %s already tracked for provider %s' % (template_name, provider_key) continue try: trackerbot.mark_provider_template(api, provider, template, **usable) print 'Added %s template %s on provider %s (datestamp: %s)' % ( group_name, template_name, provider_key, datestamp) except SlumberHttpBaseException as ex: print ex.response.status_code, ex.content # Remove provider relationships where they no longer exist, skipping unresponsive providers, # and providers not known to this environment for pt in api.providertemplate.get(limit=0)['objects']: provider_key, template_name = pt['provider']['key'], pt['template']['name'] if provider_key not in template_providers[template_name] \ and provider_key not in unresponsive_providers: if provider_key in all_providers: print "Cleaning up template %s on %s" % (template_name, provider_key) trackerbot.delete_provider_template(api, provider_key, template_name) else: print "Skipping template cleanup %s on unknown provider %s" % ( template_name, provider_key) # Remove templates that aren't on any providers anymore for template in api.template.get(limit=0)['objects']: if not template['providers']: print "Deleting template %s (no providers)" % template['name'] api.template(template['name']).delete()
import fauxfactory import traceback import dockerbot import json import requests from utils.conf import docker as docker_conf from utils.appliance import Appliance from utils.trackerbot import api from utils.log import create_logger from slumber.exceptions import HttpClientError token = docker_conf['gh_token'] owner = docker_conf['gh_owner'] repo = docker_conf['gh_repo'] tapi = api() CONT_LIMIT = docker_conf['workers'] DEBUG = docker_conf.get('debug', False) logger = create_logger('check_prs', 'prt.log') def perform_request(url): """ Simple function to assist in performing GET requests from github Runs if there is a token, else just spits out a blank dict Args: url: The url to process, which is anything after the "...repos/"
def main(trackerbot_url, mark_usable=None): api = trackerbot.api(trackerbot_url) thread_q = [] thread_lock = Lock() template_providers = defaultdict(list) all_providers = set(list_provider_keys()) unresponsive_providers = set() # Queue up list_template calls for provider_key in all_providers: ipaddress = cfme_data['management_systems'][provider_key].get( 'ipaddress', None) if ipaddress and not net.is_pingable(ipaddress): continue thread = Thread(target=get_provider_templates, args=(provider_key, template_providers, unresponsive_providers, thread_lock)) thread_q.append(thread) thread.start() # Join the queued calls for thread in thread_q: thread.join() seen_templates = set() if mark_usable is None: usable = {} else: usable = {'usable': mark_usable} existing_provider_templates = [ pt['id'] for pt in trackerbot.depaginate( api, api.providertemplate.get())['objects'] ] # Find some templates and update the API for template_name, providers in template_providers.items(): template_name = str(template_name) group_name, datestamp, stream = trackerbot.parse_template( template_name) # Don't want sprout templates if group_name in ('sprout', 'rhevm-internal'): print('Ignoring {} from group {}'.format(template_name, group_name)) continue seen_templates.add(template_name) group = trackerbot.Group(group_name, stream=stream) template = trackerbot.Template(template_name, group, datestamp) for provider_key in providers: provider = trackerbot.Provider(provider_key) if '{}_{}'.format(template_name, provider_key) in existing_provider_templates: print('Template {} already tracked for provider {}'.format( template_name, provider_key)) continue try: trackerbot.mark_provider_template(api, provider, template, **usable) print('Added {} template {} on provider {} (datestamp: {})'. format(group_name, template_name, provider_key, datestamp)) except SlumberHttpBaseException as ex: print("{}\t{}".format(ex.response.status_code, ex.content)) # Remove provider relationships where they no longer exist, skipping unresponsive providers, # and providers not known to this environment for pt in trackerbot.depaginate(api, api.providertemplate.get())['objects']: provider_key, template_name = pt['provider']['key'], pt['template'][ 'name'] if provider_key not in template_providers[template_name] \ and provider_key not in unresponsive_providers: if provider_key in all_providers: print("Cleaning up template {} on {}".format( template_name, provider_key)) trackerbot.delete_provider_template(api, provider_key, template_name) else: print("Skipping template cleanup {} on unknown provider {}". format(template_name, provider_key)) # Remove templates that aren't on any providers anymore for template in trackerbot.depaginate(api, api.template.get())['objects']: if not template['providers']: print("Deleting template {} (no providers)".format( template['name'])) api.template(template['name']).delete()