def valid_builder(buildername): """Determine if the builder you're trying to trigger is valid.""" builders = query_builders() if buildername in builders: LOG.debug("Buildername %s is valid." % buildername) return True else: LOG.warning("Buildername %s is *NOT* valid." % buildername) LOG.info("Check the file we just created builders.txt for " "a list of valid builders.") with open(path_to_file("builders.txt"), "wb") as fd: for b in sorted(builders): fd.write(b + "\n") return False
def _fetch_file(filename): """ Helper method to download files. This function caches the uncompressed gzip files requested in the past. Returns all jobs inside of this buildjson file. """ url = "%s/%s.gz" % (BUILDJSON_DATA, filename) # If the file exists and is valid we won't download it again fetch_file(filename, url) LOG.debug("About to load %s." % filename) builds = json.load(open(path_to_file(filename)))["builds"] return builds
def valid_builder(buildername): """Determine if the builder you're trying to trigger is valid.""" builders = query_builders() if buildername in builders: LOG.debug("Buildername %s is valid." % buildername) return True else: LOG.warning("Buildername %s is *NOT* valid." % buildername) LOG.info("Check the file we just created builders.txt for " "a list of valid builders.") with open(path_to_file('builders.txt'), "wb") as fd: for b in sorted(builders): fd.write(b + "\n") return False
def _fetch_data(filename): """ Helper method to fetch the buildjson data we need. This function caches the uncompressed gzip files requested in the past. Returns all jobs inside of this buildjson file. """ url = "%s/%s.gz" % (BUILDJSON_DATA, filename) if not os.path.isabs(filename): filepath = path_to_file(filename) else: filepath = filename # If the file exists and is valid we won't download it again json_contents = load_file(filepath, url) return json_contents["builds"]
def setup_logging(level=logging.INFO): """ Save every message (including debug ones) to ~/.mozilla/mozci/mozci-debug.log. Log messages of level equal or greater then 'level' to the terminal. As seen in: https://docs.python.org/2/howto/logging-cookbook.html#logging-to-multiple-destinations """ global LOG if LOG: return LOG # We need to set the root logger or we will not see messages from dependent # modules LOG = logging.getLogger() # Handler 1 - Store all debug messages in a specific file logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s:\t %(message)s', datefmt='%m/%d/%Y %I:%M:%S', filename=path_to_file('mozci-debug.log'), filemode='w') # Handler 2 - Console output console = logging.StreamHandler() console.setLevel(level) # console does not use the same formatter specified in basicConfig # we have to set it again formatter = logging.Formatter( '%(asctime)s %(name)s %(levelname)s:\t %(message)s', datefmt='%m/%d/%Y %I:%M:%S') console.setFormatter(formatter) LOG.addHandler(console) LOG.info("Setting %s level" % logging.getLevelName(level)) if level != logging.DEBUG: # requests is too noisy and adds no value logging.getLogger("requests").setLevel(logging.WARNING) return LOG
def setup_logging(level=logging.INFO): """ Save every message (including debug ones) to ~/.mozilla/mozci/mozci-debug.log. Log messages of level equal or greater then 'level' to the terminal. As seen in: https://docs.python.org/2/howto/logging-cookbook.html#logging-to-multiple-destinations """ global LOG if LOG: return LOG # We need to set the root logger or we will not see messages from dependent # modules LOG = logging.getLogger() # Handler 1 - Store all debug messages in a specific file logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s:\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S', filename=path_to_file('mozci-debug.log'), filemode='w') # Handler 2 - Console output console = logging.StreamHandler() console.setLevel(level) # console does not use the same formatter specified in basicConfig # we have to set it again formatter = logging.Formatter('%(asctime)s %(name)s %(levelname)s:\t%(message)s', datefmt='%m/%d/%Y %I:%M:%S') console.setFormatter(formatter) LOG.addHandler(console) LOG.info("Setting %s level" % logging.getLevelName(level)) if level != logging.DEBUG: # requests is too noisy and adds no value logging.getLogger("requests").setLevel(logging.WARNING) return LOG
def _fetch_data(filename): """ Helper method to fetch the buildjson data we need. This function caches the uncompressed gzip files requested in the past. Returns all jobs inside of this buildjson file. """ global BUILDS_CACHE if filename in BUILDS_CACHE: return BUILDS_CACHE[filename] url = "%s/%s.gz" % (BUILDJSON_DATA, filename) if not os.path.isabs(filename): filepath = path_to_file(filename) else: filepath = filename # If the file exists and is valid we won't download it again json_contents = load_file(filepath, url) BUILDS_CACHE[filename] = json_contents["builds"] return json_contents["builds"]
"""This script writes a mapping from platforms to tests that run in it to graph.json.""" import ujson as json from mozci.platforms import build_tests_per_platform_graph, _filter_builders_matching from mozci.sources.allthethings import fetch_allthethings_data from mozci.utils.transfer import path_to_file if __name__ == '__main__': with open(path_to_file('graph.json'), 'w') as f: builders = _filter_builders_matching(fetch_allthethings_data()['builders'], " try ") graph = build_tests_per_platform_graph(builders) json.dump(graph, f, sort_keys=True, indent=4, separators=(',', ': '))
""" This script is designed as a way to determine meta-data about repositories. """ from __future__ import absolute_import import json import logging import os from buildapi_client import BuildapiAuthError, make_query_repositories_request from mozci.errors import AuthenticationError, MozciError from mozci.utils.authentication import get_credentials, remove_credentials from mozci.utils.transfer import path_to_file LOG = logging.getLogger('mozci') REPOSITORIES_FILE = path_to_file("repositories.txt") REPOSITORIES = {} # # Functions to query # def query_repository(repo_name): """ Return dictionary with information about a specific repository. Raises MozciError if the repository does not exist. """ repositories = query_repositories() if repo_name not in repositories: repositories = query_repositories(clobber=True)
def setup_logging(level=logging.INFO, datefmt='%I:%M:%S', show_timestamps=True, show_name_level=False, requests_output=False): """ It helps set up mozci's logging and makes it easy to customize. It returns a cached logger if already called once. By default: * It logs INFO messages * It sets the default datefmt * It sets to show the timestamps of the messages * It does not show the messages level name (e.g. 'DEBUG') * It mutes INFO messages of the requests package since it is noisy * It logs messages of level equal or greater than 'level' to the terminal. * It also saves every message (including debug ones) to ~/.mozilla/mozci/mozci-debug.log. :param level: It sets which level messages to log :type level: int :param datefmt: It sets the format of the timestamps :type datefmt: str :param show_timestamps: It determines if to show the timestamps :type show_timestamps: bool :param show_name_level: It determines if to show the level name :type show_name_level: bool :param requests_output: It determines if to show logging of requests below the WARNING level :type requests_output: bool :returns: cached logger :rtype: logging.LOGGER As seen in: https://docs.python.org/2/howto/logging-cookbook.html#logging-to-multiple-destinations """ global LOG if LOG: return LOG # We need to set the root logger or we will not see messages from dependent # modules LOG = logging.getLogger() format = '' if show_timestamps: format += '%(asctime)s ' format += '%(name)s' if show_name_level: format += ' %(levelname)s ' format += '\t%(message)s' # Handler 1 - Store all debug messages in a specific file logging.basicConfig(level=logging.DEBUG, format=format, datefmt=datefmt, filename=path_to_file('mozci-debug.log'), filemode='w') # Handler 2 - Console output console = logging.StreamHandler() console.setLevel(level) # console does not use the same formatter specified in basicConfig # we have to set it again formatter = logging.Formatter(format, datefmt=datefmt) console.setFormatter(formatter) LOG.addHandler(console) LOG.info("Setting %s level" % logging.getLevelName(level)) if not requests_output: # requests is too noisy and adds no value # Set the value to warning to show actual issues logging.getLogger("requests").setLevel(logging.WARNING) return LOG
#! /usr/bin/env python """ This script is designed as a way to determine meta-data about repositories. """ from __future__ import absolute_import import json import logging import os from thclient import TreeherderClient from mozci.errors import MozciError from mozci.utils.transfer import path_to_file LOG = logging.getLogger("mozci") REPOSITORIES_FILE = path_to_file("repositories.txt") REPOSITORIES = {} TREEHERDER_URL = "treeherder.mozilla.org" # # Functions to query # def query_repository(repo_name): """ Return dictionary with information about a specific repository. Raises MozciError if the repository does not exist. """ repositories = query_repositories() if repo_name not in repositories:
}, * **master_builders** * **slavepools** """ import json import logging import os import requests from mozci.utils.transfer import path_to_file LOG = logging.getLogger('mozci') FILENAME = path_to_file("allthethings.json") ALLTHETHINGS = \ "https://secure.pub.build.mozilla.org/builddata/reports/allthethings.json" DATA = None def fetch_allthethings_data(no_caching=False, verify=True): """ It fetches the allthethings.json file. If no_caching is True, we fetch it every time without creating a file. If verify is False, we load from disk without checking. This should only be used if allthethings.json exists and it's trusted. """ def _fetch():
"""This script writes a mapping from platforms to tests that run in it to graph.json.""" import json from mozci.platforms import build_tests_per_platform_graph, list_builders from mozci.utils.transfer import path_to_file if __name__ == '__main__': with open(path_to_file('graph.json'), 'w') as f: graph = build_tests_per_platform_graph(builders=list_builders( repo_name='try')) json.dump(graph, f, sort_keys=True, indent=4, separators=(',', ': '))
# License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # ***** END LICENSE BLOCK ***** """Module for http authentication operations.""" import getpass import logging import os import keyring import requests from mozci.utils.transfer import path_to_file AUTH = None CREDENTIALS_PATH = path_to_file("credentials.cfg") DIRNAME = os.path.dirname(CREDENTIALS_PATH) KEYRING_KEY = 'ldap' # We use buildapi since we don't have a better option LDAP_HOST = 'https://secure.pub.build.mozilla.org/buildapi/self-serve' LOG = logging.getLogger('mozci') def _prompt_password_storing(https_username): https_password = getpass.getpass() store_password = raw_input( "Do you want to store your password in encrypted form (y or n)? ") if store_password == "y": keyring.set_password(KEYRING_KEY, https_username, https_password)
# You can obtain one at http://mozilla.org/MPL/2.0/. # ***** END LICENSE BLOCK ***** """Module for http authentication operations.""" import getpass import logging import os import keyring import requests from mozci.utils.transfer import path_to_file AUTH = None CREDENTIALS_PATH = path_to_file("credentials.cfg") DIRNAME = os.path.dirname(CREDENTIALS_PATH) KEYRING_KEY = 'ldap' # We use buildapi since we don't have a better option LDAP_HOST = 'https://secure.pub.build.mozilla.org/buildapi/self-serve' LOG = logging.getLogger('mozci') def _prompt_password_storing(https_username): https_password = getpass.getpass() store_password = raw_input( "Do you want to store your password in encrypted form (y or n)? ") if store_password == "y": keyring.set_password(KEYRING_KEY, https_username, https_password) else: