def valid_builder(buildername):
"""Determine if the builder you're trying to trigger is valid."""
builders = query_builders()
if buildername in builders:
LOG.debug("Buildername %s is valid." % buildername)
return True
else:
LOG.warning("Buildername %s is *NOT* valid." % buildername)
LOG.info("Check the file we just created builders.txt for " "a list of valid builders.")
with open(path_to_file("builders.txt"), "wb") as fd:
for b in sorted(builders):
fd.write(b + "\n")
return False
def _fetch_file(filename):
"""
Helper method to download files.
This function caches the uncompressed gzip files requested in the past.
Returns all jobs inside of this buildjson file.
"""
url = "%s/%s.gz" % (BUILDJSON_DATA, filename)
# If the file exists and is valid we won't download it again
fetch_file(filename, url)
LOG.debug("About to load %s." % filename)
builds = json.load(open(path_to_file(filename)))["builds"]
return builds
def valid_builder(buildername):
"""Determine if the builder you're trying to trigger is valid."""
builders = query_builders()
if buildername in builders:
LOG.debug("Buildername %s is valid." % buildername)
return True
else:
LOG.warning("Buildername %s is *NOT* valid." % buildername)
LOG.info("Check the file we just created builders.txt for "
"a list of valid builders.")
with open(path_to_file('builders.txt'), "wb") as fd:
for b in sorted(builders):
fd.write(b + "\n")
return False
def _fetch_data(filename):
"""
Helper method to fetch the buildjson data we need.
This function caches the uncompressed gzip files requested in the past.
Returns all jobs inside of this buildjson file.
"""
url = "%s/%s.gz" % (BUILDJSON_DATA, filename)
if not os.path.isabs(filename):
filepath = path_to_file(filename)
else:
filepath = filename
# If the file exists and is valid we won't download it again
json_contents = load_file(filepath, url)
return json_contents["builds"]
def setup_logging(level=logging.INFO):
"""
Save every message (including debug ones) to ~/.mozilla/mozci/mozci-debug.log.
Log messages of level equal or greater then 'level' to the terminal.
As seen in:
https://docs.python.org/2/howto/logging-cookbook.html#logging-to-multiple-destinations
"""
global LOG
if LOG:
return LOG
# We need to set the root logger or we will not see messages from dependent
# modules
LOG = logging.getLogger()
# Handler 1 - Store all debug messages in a specific file
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s:\t %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
filename=path_to_file('mozci-debug.log'),
filemode='w')
# Handler 2 - Console output
console = logging.StreamHandler()
console.setLevel(level)
# console does not use the same formatter specified in basicConfig
# we have to set it again
formatter = logging.Formatter(
'%(asctime)s %(name)s %(levelname)s:\t %(message)s',
datefmt='%m/%d/%Y %I:%M:%S')
console.setFormatter(formatter)
LOG.addHandler(console)
LOG.info("Setting %s level" % logging.getLevelName(level))
if level != logging.DEBUG:
# requests is too noisy and adds no value
logging.getLogger("requests").setLevel(logging.WARNING)
return LOG
def setup_logging(level=logging.INFO):
"""
Save every message (including debug ones) to ~/.mozilla/mozci/mozci-debug.log.
Log messages of level equal or greater then 'level' to the terminal.
As seen in:
https://docs.python.org/2/howto/logging-cookbook.html#logging-to-multiple-destinations
"""
global LOG
if LOG:
return LOG
# We need to set the root logger or we will not see messages from dependent
# modules
LOG = logging.getLogger()
# Handler 1 - Store all debug messages in a specific file
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s:\t%(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
filename=path_to_file('mozci-debug.log'),
filemode='w')
# Handler 2 - Console output
console = logging.StreamHandler()
console.setLevel(level)
# console does not use the same formatter specified in basicConfig
# we have to set it again
formatter = logging.Formatter('%(asctime)s %(name)s %(levelname)s:\t%(message)s',
datefmt='%m/%d/%Y %I:%M:%S')
console.setFormatter(formatter)
LOG.addHandler(console)
LOG.info("Setting %s level" % logging.getLevelName(level))
if level != logging.DEBUG:
# requests is too noisy and adds no value
logging.getLogger("requests").setLevel(logging.WARNING)
return LOG
def _fetch_data(filename):
"""
Helper method to fetch the buildjson data we need.
This function caches the uncompressed gzip files requested in the past.
Returns all jobs inside of this buildjson file.
"""
global BUILDS_CACHE
if filename in BUILDS_CACHE:
return BUILDS_CACHE[filename]
url = "%s/%s.gz" % (BUILDJSON_DATA, filename)
if not os.path.isabs(filename):
filepath = path_to_file(filename)
else:
filepath = filename
# If the file exists and is valid we won't download it again
json_contents = load_file(filepath, url)
BUILDS_CACHE[filename] = json_contents["builds"]
return json_contents["builds"]
"""This script writes a mapping from platforms to tests that run in it to graph.json."""
import ujson as json
from mozci.platforms import build_tests_per_platform_graph, _filter_builders_matching
from mozci.sources.allthethings import fetch_allthethings_data
from mozci.utils.transfer import path_to_file
if __name__ == '__main__':
with open(path_to_file('graph.json'), 'w') as f:
builders = _filter_builders_matching(fetch_allthethings_data()['builders'], " try ")
graph = build_tests_per_platform_graph(builders)
json.dump(graph, f, sort_keys=True, indent=4, separators=(',', ': '))
"""
This script is designed as a way to determine meta-data about repositories.
"""
from __future__ import absolute_import
import json
import logging
import os
from buildapi_client import BuildapiAuthError, make_query_repositories_request
from mozci.errors import AuthenticationError, MozciError
from mozci.utils.authentication import get_credentials, remove_credentials
from mozci.utils.transfer import path_to_file
LOG = logging.getLogger('mozci')
REPOSITORIES_FILE = path_to_file("repositories.txt")
REPOSITORIES = {}
#
# Functions to query
#
def query_repository(repo_name):
"""
Return dictionary with information about a specific repository.
Raises MozciError if the repository does not exist.
"""
repositories = query_repositories()
if repo_name not in repositories:
repositories = query_repositories(clobber=True)
def setup_logging(level=logging.INFO, datefmt='%I:%M:%S', show_timestamps=True,
show_name_level=False, requests_output=False):
""" It helps set up mozci's logging and makes it easy to customize.
It returns a cached logger if already called once.
By default:
* It logs INFO messages
* It sets the default datefmt
* It sets to show the timestamps of the messages
* It does not show the messages level name (e.g. 'DEBUG')
* It mutes INFO messages of the requests package since it is noisy
* It logs messages of level equal or greater than 'level' to the terminal.
* It also saves every message (including debug ones) to ~/.mozilla/mozci/mozci-debug.log.
:param level: It sets which level messages to log
:type level: int
:param datefmt: It sets the format of the timestamps
:type datefmt: str
:param show_timestamps: It determines if to show the timestamps
:type show_timestamps: bool
:param show_name_level: It determines if to show the level name
:type show_name_level: bool
:param requests_output: It determines if to show logging of requests below the WARNING level
:type requests_output: bool
:returns: cached logger
:rtype: logging.LOGGER
As seen in:
https://docs.python.org/2/howto/logging-cookbook.html#logging-to-multiple-destinations
"""
global LOG
if LOG:
return LOG
# We need to set the root logger or we will not see messages from dependent
# modules
LOG = logging.getLogger()
format = ''
if show_timestamps:
format += '%(asctime)s '
format += '%(name)s'
if show_name_level:
format += ' %(levelname)s '
format += '\t%(message)s'
# Handler 1 - Store all debug messages in a specific file
logging.basicConfig(level=logging.DEBUG,
format=format,
datefmt=datefmt,
filename=path_to_file('mozci-debug.log'),
filemode='w')
# Handler 2 - Console output
console = logging.StreamHandler()
console.setLevel(level)
# console does not use the same formatter specified in basicConfig
# we have to set it again
formatter = logging.Formatter(format, datefmt=datefmt)
console.setFormatter(formatter)
LOG.addHandler(console)
LOG.info("Setting %s level" % logging.getLevelName(level))
if not requests_output:
# requests is too noisy and adds no value
# Set the value to warning to show actual issues
logging.getLogger("requests").setLevel(logging.WARNING)
return LOG
#! /usr/bin/env python
"""
This script is designed as a way to determine meta-data about repositories.
"""
from __future__ import absolute_import
import json
import logging
import os
from thclient import TreeherderClient
from mozci.errors import MozciError
from mozci.utils.transfer import path_to_file
LOG = logging.getLogger("mozci")
REPOSITORIES_FILE = path_to_file("repositories.txt")
REPOSITORIES = {}
TREEHERDER_URL = "treeherder.mozilla.org"
#
# Functions to query
#
def query_repository(repo_name):
"""
Return dictionary with information about a specific repository.
Raises MozciError if the repository does not exist.
"""
repositories = query_repositories()
if repo_name not in repositories:
},
* **master_builders**
* **slavepools**
"""
import json
import logging
import os
import requests
from mozci.utils.transfer import path_to_file
LOG = logging.getLogger('mozci')
FILENAME = path_to_file("allthethings.json")
ALLTHETHINGS = \
"https://secure.pub.build.mozilla.org/builddata/reports/allthethings.json"
DATA = None
def fetch_allthethings_data(no_caching=False, verify=True):
"""
It fetches the allthethings.json file.
If no_caching is True, we fetch it every time without creating a file.
If verify is False, we load from disk without checking. This should only be used if
allthethings.json exists and it's trusted.
"""
def _fetch():
"""This script writes a mapping from platforms to tests that run in it to graph.json."""
import json
from mozci.platforms import build_tests_per_platform_graph, list_builders
from mozci.utils.transfer import path_to_file
if __name__ == '__main__':
with open(path_to_file('graph.json'), 'w') as f:
graph = build_tests_per_platform_graph(builders=list_builders(
repo_name='try'))
json.dump(graph, f, sort_keys=True, indent=4, separators=(',', ': '))
},
* **master_builders**
* **slavepools**
"""
import json
import logging
import os
import requests
from mozci.utils.transfer import path_to_file
LOG = logging.getLogger('mozci')
FILENAME = path_to_file("allthethings.json")
ALLTHETHINGS = \
"https://secure.pub.build.mozilla.org/builddata/reports/allthethings.json"
DATA = None
def fetch_allthethings_data(no_caching=False, verify=True):
"""
It fetches the allthethings.json file.
If no_caching is True, we fetch it every time without creating a file.
If verify is False, we load from disk without checking. This should only be used if
allthethings.json exists and it's trusted.
"""
def _fetch():
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""Module for http authentication operations."""
import getpass
import logging
import os
import keyring
import requests
from mozci.utils.transfer import path_to_file
AUTH = None
CREDENTIALS_PATH = path_to_file("credentials.cfg")
DIRNAME = os.path.dirname(CREDENTIALS_PATH)
KEYRING_KEY = 'ldap'
# We use buildapi since we don't have a better option
LDAP_HOST = 'https://secure.pub.build.mozilla.org/buildapi/self-serve'
LOG = logging.getLogger('mozci')
def _prompt_password_storing(https_username):
https_password = getpass.getpass()
store_password = raw_input(
"Do you want to store your password in encrypted form (y or n)? ")
if store_password == "y":
keyring.set_password(KEYRING_KEY, https_username, https_password)
def setup_logging(level=logging.INFO,
datefmt='%I:%M:%S',
show_timestamps=True,
show_name_level=False,
requests_output=False):
""" It helps set up mozci's logging and makes it easy to customize.
It returns a cached logger if already called once.
By default:
* It logs INFO messages
* It sets the default datefmt
* It sets to show the timestamps of the messages
* It does not show the messages level name (e.g. 'DEBUG')
* It mutes INFO messages of the requests package since it is noisy
* It logs messages of level equal or greater than 'level' to the terminal.
* It also saves every message (including debug ones) to ~/.mozilla/mozci/mozci-debug.log.
:param level: It sets which level messages to log
:type level: int
:param datefmt: It sets the format of the timestamps
:type datefmt: str
:param show_timestamps: It determines if to show the timestamps
:type show_timestamps: bool
:param show_name_level: It determines if to show the level name
:type show_name_level: bool
:param requests_output: It determines if to show logging of requests below the WARNING level
:type requests_output: bool
:returns: cached logger
:rtype: logging.LOGGER
As seen in:
https://docs.python.org/2/howto/logging-cookbook.html#logging-to-multiple-destinations
"""
global LOG
if LOG:
return LOG
# We need to set the root logger or we will not see messages from dependent
# modules
LOG = logging.getLogger()
format = ''
if show_timestamps:
format += '%(asctime)s '
format += '%(name)s'
if show_name_level:
format += ' %(levelname)s '
format += '\t%(message)s'
# Handler 1 - Store all debug messages in a specific file
logging.basicConfig(level=logging.DEBUG,
format=format,
datefmt=datefmt,
filename=path_to_file('mozci-debug.log'),
filemode='w')
# Handler 2 - Console output
console = logging.StreamHandler()
console.setLevel(level)
# console does not use the same formatter specified in basicConfig
# we have to set it again
formatter = logging.Formatter(format, datefmt=datefmt)
console.setFormatter(formatter)
LOG.addHandler(console)
LOG.info("Setting %s level" % logging.getLevelName(level))
if not requests_output:
# requests is too noisy and adds no value
# Set the value to warning to show actual issues
logging.getLogger("requests").setLevel(logging.WARNING)
return LOG
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""Module for http authentication operations."""
import getpass
import logging
import os
import keyring
import requests
from mozci.utils.transfer import path_to_file
AUTH = None
CREDENTIALS_PATH = path_to_file("credentials.cfg")
DIRNAME = os.path.dirname(CREDENTIALS_PATH)
KEYRING_KEY = 'ldap'
# We use buildapi since we don't have a better option
LDAP_HOST = 'https://secure.pub.build.mozilla.org/buildapi/self-serve'
LOG = logging.getLogger('mozci')
def _prompt_password_storing(https_username):
https_password = getpass.getpass()
store_password = raw_input(
"Do you want to store your password in encrypted form (y or n)? ")
if store_password == "y":
keyring.set_password(KEYRING_KEY, https_username, https_password)
else:
