def new_session(insecure=False, ca_file=None, total_retries=None):
session = requests.Session()
if total_retries is not None:
http_adapter = adapters.HTTPAdapter(max_retries=retry.Retry(
total=total_retries))
https_adapter = adapters.HTTPAdapter(max_retries=retry.Retry(
total=total_retries))
session.mount('http://', http_adapter)
session.mount('https://', https_adapter)
session.verify = ca_file if ca_file else not insecure
return session
def requests_with_retry(retries: int = 3) -> requests.Session:
session = requests.Session()
adapter = requests.adapters.HTTPAdapter(max_retries=retry.Retry(
total=retries, backoff_factor=0.1, status_forcelist=(500, 502, 504)))
session.mount("http://", adapter)
session.mount("https://", adapter)
return session
def __init__(self, base_url: str, *args, **kwargs):
"""
Creates a new HTTP session instance.
Args:
base_url: The base URL for this HTTP session
*args: The optional positional arguments
**kwargs: The optional keyword arguments
"""
super().__init__(*args, **kwargs)
self._base_url = base_url
# Custom headers
self.headers.update({
'User-Agent':
'geodatabr/{version} ({url})'.format(version=__meta__.__version__,
url=__meta__.__url__),
})
# Automatic retries
for protocol in ('http://', 'https://'):
self.mount(
protocol,
adapters.HTTPAdapter(max_retries=retry.Retry(
total=HTTP_MAX_RETRIES,
backoff_factor=HTTP_BACKOFF_FACTOR,
status_forcelist=HTTP_RETRY_STATUSES)))
def _make_session(self, max_retries, retry_backoff, auth_url,
endpoint):
"""Create requests session that retries on connection failures"""
if max_retries is None:
max_retries = constants.DEFAULT_RETRIES
if retry_backoff is None:
retry_backoff = constants.DEFAULT_RETRY_BACKOFF
session = requests.Session()
# this makes it retry on timeouts, connection errors,
# and 503 Unavailable responses
adapter = requests.adapters.HTTPAdapter(
max_retries=requests_retry.Retry(
total=max(0, max_retries),
backoff_factor=max(0, retry_backoff),
status_forcelist=set([503]),
method_whitelist=set(),
)
)
# Retries for authentication and API requests
session.mount(auth_url, adapter)
session.mount(endpoint, adapter)
return session
def reconcile_rmq_vhost(): # pragma: no cover
url = settings.RMQ_API_ENDPOINT
vhost_name = settings.RMQ_VHOST
user = settings.RMQ_USER
userPass = user + ":" + settings.RMQ_PASSWORD
authHeader = "Basic " + base64.b64encode(userPass.encode()).decode("utf-8")
payload = json.dumps({
"vhosts": [{
"name": vhost_name
}],
"permissions": [{
"user": user,
"vhost": vhost_name,
"configure": ".*",
"write": ".*",
"read": ".*"
}],
"policies": [{
"vhost": vhost_name,
"name": "sync-nodes-" + vhost_name,
"pattern": ".*",
"apply-to": "all",
"definition": {
"ha-sync-mode": "automatic",
"ha-mode": "all"
},
"priority": 0
}]
})
headers = {
'authorization': authHeader,
'content-type': "application/json",
}
requests_session = requests.Session()
parsed_url = urlparse(url)
retry_adapter = requests.adapters.HTTPAdapter(
max_retries=requests_retry.Retry(
total=400,
backoff_factor=0.5,
method_whitelist=set(['POST']),
))
requests_session.mount('{scheme}://'.format(scheme=parsed_url.scheme),
retry_adapter)
response = requests_session.request("POST",
url,
data=payload,
headers=headers)
if response.ok:
print('Created vhost and permissions for: {vhost}'.format(
vhost=vhost_name))
else:
print(
"Could not create vhost {vhost} on RabbitMQ management API endpoint {api_endpoint}: {reason}"
.format(vhost=vhost_name, api_endpoint=url,
reason=response.reason))
response.raise_for_status()
def _make_session(self):
session = requests.Session()
retry_conf = retry.Retry(total=self.tries, connect=self.tries,
read=self.tries, backoff_factor=1,
status_forcelist=[429])
retry_conf.BACKOFF_MAX = 8
adapter = requests.adapters.HTTPAdapter(max_retries=retry_conf)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
def requests_context():
session = requests.Session()
retry = urllib_retry.Retry(total=10,
backoff_factor=1,
status_forcelist=(500, 502, 503, 504))
adapter = requests_adapters.HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
def _retrying_request_session(retries=10, backoff_factor=0.5):
"""Returns a requests session with retry logic."""
retry_config = retry.Retry(total=retries,
connect=retries,
read=retries,
method_whitelist=['GET', 'POST'],
backoff_factor=backoff_factor)
adapter = adapters.HTTPAdapter(max_retries=retry_config)
session = requests.Session()
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
class HTTPRequest:
"""
A requests wrapper preconfigured with best practice timeout and retry policies and per-thread session tracking.
The timeout and retry policies are configured to turn on exponential backoff support provided by urllib3, and to
make the client resistant to both network issues and intermittent server-side errors.
The per-thread session tracking is to avoid many threads sharing the same session in multithreaded environments.
"""
retry_policy = retry.Retry(read=4,
status=4,
backoff_factor=0.1,
status_forcelist=frozenset({500, 502, 503, 504}))
timeout_policy = timeout.Timeout(connect=20, read=40)
codes = requests.codes
def __init__(self, max_redirects=1024, obey_retry_after=True):
self.sessions = {}
self.max_redirects = max_redirects
self.obey_retry_after = obey_retry_after
def __call__(self, *args, **kwargs):
if get_ident() not in self.sessions:
session = Session()
session.max_redirects = self.max_redirects
session.obey_retry_after = self.obey_retry_after
adapter = HTTPAdapter(max_retries=self.retry_policy)
session.mount('http://', adapter)
session.mount('https://', adapter)
self.sessions[get_ident()] = session
return self.sessions[get_ident()].request(*args, timeout=self.timeout_policy, **kwargs)
def head(self, url, *args, **kwargs):
return self(url=url, method="HEAD", *args, **kwargs)
def get(self, url, *args, **kwargs):
return self(url=url, method="GET", *args, **kwargs)
def put(self, url, *args, **kwargs):
return self(url=url, method="PUT", *args, **kwargs)
def post(self, url, *args, **kwargs):
return self(url=url, method="POST", *args, **kwargs)
def delete(self, url, *args, **kwargs):
return self(url=url, method="DELETE", *args, **kwargs)
def patch(self, url, *args, **kwargs):
return self(url=url, method="PATCH", *args, **kwargs)
def options(self, url, *args, **kwargs):
return self(url=url, method="OPTIONS", *args, **kwargs)
def __init__(self):
self.bugzilla = hotness.bz.Bugzilla(consumer=self,
config=hotness_config["bugzilla"])
self.buildsys = hotness.buildsys.Koji(consumer=self,
config=hotness_config["koji"])
self.pdc_url = hotness_config["pdc_url"]
self.dist_git_url = hotness_config["dist_git_url"]
self.anitya_url = hotness_config["anitya"]["url"]
# Also, set up our global cache object.
_log.info("Configuring cache.")
with hotness.cache.cache_lock:
if not hotness.cache.cache.is_configured:
hotness.cache.cache.configure(**hotness_config["cache"])
self.mdapi_url = hotness_config["mdapi_url"]
_log.info("Using hotness.mdapi_url=%r" % self.mdapi_url)
self.repoid = hotness_config["repoid"]
_log.info("Using hotness.repoid=%r" % self.repoid)
self.distro = hotness_config["distro"]
_log.info("Using hotness.distro=%r" % self.distro)
# Retrieve the requests configuration; by default requests time out
# after 15 seconds and are retried up to 3 times.
self.requests_session = requests.Session()
self.timeout = (
hotness_config["connect_timeout"],
hotness_config["read_timeout"],
)
retries = hotness_config["requests_retries"]
retry_conf = retry.Retry(total=retries,
connect=retries,
read=retries,
backoff_factor=1)
retry_conf.BACKOFF_MAX = 5
self.requests_session.mount(
"http://", requests.adapters.HTTPAdapter(max_retries=retry_conf))
self.requests_session.mount(
"https://", requests.adapters.HTTPAdapter(max_retries=retry_conf))
_log.info("Requests timeouts are {}s (connect) and {}s (read)"
" with {} retries".format(self.timeout[0], self.timeout[1],
retries))
# Build a little store where we'll keep track of what koji scratch
# builds we have kicked off. We'll look later for messages indicating
# that they have completed.
self.scratch_builds = {}
_log.info("That new hotness ticket filer is all initialized")
def requests_with_retry(retries: int = 3) -> requests.Session:
"""Creates a `requests` object. Request will be attempted `retries` times.
Will retry on 500, 502 and 504 status codes.
>>> response = requests_with_retry(3).get("http://someurl.com")
"""
session = requests.Session()
adapter = requests.adapters.HTTPAdapter(max_retries=retry.Retry(
total=retries,
backoff_factor=0.1,
status_forcelist=(500, 502, 504),
), )
session.mount("http://", adapter)
session.mount("https://", adapter)
return session
def _get_session(self) -> requests.Session:
"""
Initialize and return a session allows make retry when some errors
will raised.
"""
session = requests.Session()
retry = rq_retry.Retry(
total=self.retries_number,
read=self.retries_number,
connect=self.retries_number,
backoff_factor=self.backoff_factor,
status_forcelist=self.status_forcelist,
)
adapter = rq_adapt.HTTPAdapter(max_retries=retry)
session.mount("https://", adapter)
return session
def __init__(self,
config,
event_listener=None,
tries=DEFAULT_TRIES,
session=None):
"""
:param config: downloader configuration
:type config: nectar.config.DownloaderConfig
:param event_listener: event listener providing life-cycly callbacks
:type event_listener: nectar.listener.DownloadEventListener
:param tries: total number of requests made to the remote server,
including first unsuccessful one
:type tries: int
:param session: The requests Session to use when downloaded. If one
is not provided, one will be created and used for the
lifetime of this downloader.
:type session: requests.Session
"""
super(HTTPThreadedDownloader, self).__init__(config, event_listener)
# throttling support
self._bytes_lock = threading.RLock()
self._bytes_this_second = 0
self._time_bytes_this_second_was_cleared = datetime.datetime.now()
# thread-safety when firing events
self._event_lock = threading.RLock()
# default tries to fetch item
self.tries = tries
# set of locations that produced a connection error
self.failed_netlocs = set([])
self.session = session or build_session(config)
# Configure an adapter to retry failed requests. See urllib3's documentation
# for details on each argument.
retry_conf = retry.Retry(total=tries,
connect=tries,
read=tries,
backoff_factor=1)
retry_conf.BACKOFF_MAX = 8
adapter = requests.adapters.HTTPAdapter(max_retries=retry_conf)
self.session.mount('http://', adapter)
self.session.mount('https://', adapter)
def get_http():
"""Get the http object."""
ensure_dir(CLUSTERFUZZ_TESTCASES_DIR)
http = requests_cache.CachedSession(cache_name=os.path.join(
CLUSTERFUZZ_TESTCASES_DIR, 'http_cache'),
backend='sqlite',
allowable_methods=('GET', 'POST'),
allowable_codes=[200],
expire_after=HTTP_CACHE_TTL)
http.mount(
'https://',
adapters.HTTPAdapter(
# backoff_factor is 0.5. Therefore, the max wait time is 16s.
retry.Retry(total=5,
backoff_factor=0.5,
status_forcelist=[500, 502, 503, 504])))
return http
def requests_retry_session(
retries=3,
backoff_factor=0.3,
status_forcelist=(500, 502, 504),
session=None,
):
session = session or requests.Session()
max_retries = retry.Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist,
)
adapter = adapters.HTTPAdapter(max_retries=max_retries)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
def url_helper(sess=None,
retries=10,
backoff_factor=0.3,
status_forcelist=(500, 502, 504)):
"""url_helper
:param sess: ``requests.Session``
object like
.. code-block:: python
s = requests.Session()
s.auth = ('user', 'pass')
s.headers.update({'x-test': 'true'})
response = url_helper(sesssion=s).get(
'https://www.peterbe.com'
)
:param retries: number of retries
default is ``3``
:param backoff_factor: seconds per attempt
default is ``0.3``
:param status_forcelist: optional tuple list
of retry error HTTP status codes
default is ``500, 502, 504``
"""
session = sess or requests.Session()
retry = requests_retry.Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status_forcelist=status_forcelist,
)
adapter = adapters.HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
class GSClient:
base_url = "https://www.googleapis.com/storage/v1/"
presigned_url_base = "https://storage.googleapis.com/"
scope = "https://www.googleapis.com/auth/cloud-platform"
instance_metadata_url = "http://metadata.google.internal/computeMetadata/v1/"
svc_acct_token_url = instance_metadata_url + "instance/service-accounts/default/token"
project_id_metadata_url = instance_metadata_url + "project/project-id"
suppress_paging_warning = False
retry_policy = retry.Retry(connect=5,
read=5,
status_forcelist=frozenset({500, 502, 503,
504}),
backoff_factor=1)
timeout = 20
def __init__(self, config=None, **session_kwargs):
if config is None:
config = tweak.Config(__name__, save_on_exit=False)
self.config = config
self._service_jwt = None
self._oauth2_token = None
self._sessions = {}
self._session_kwargs = session_kwargs
def get_session(self):
thread_id = get_ident()
if thread_id not in self._sessions:
session = requests.Session(**self._session_kwargs)
session.headers.update({"User-Agent": self.__class__.__name__})
token = self.get_oauth2_token()
if token is not None:
session.headers.update({"Authorization": "Bearer " + token})
adapter = HTTPAdapter(max_retries=self.retry_policy)
session.mount('http://', adapter)
session.mount('https://', adapter)
self._sessions[thread_id] = session
return self._sessions[thread_id]
def get_oauth2_token(self):
# TODO: invalidate and refetch before expiration
if self._oauth2_token is None:
try:
service_jwt = self.get_service_jwt()
params = dict(
grant_type="urn:ietf:params:oauth:grant-type:jwt-bearer",
assertion=service_jwt)
res = requests.post(
"https://www.googleapis.com/oauth2/v4/token", data=params)
except NoServiceCredentials:
try:
res = requests.get(self.svc_acct_token_url,
headers={"Metadata-Flavor": "Google"})
except Exception:
logger.warn(
'API credentials not configured. Sending unsigned requests.'
)
logger.warn(
'To set credentials, run "gs configure" or set GOOGLE_APPLICATION_CREDENTIALS.'
)
return None
res.raise_for_status()
self._oauth2_token = res.json()["access_token"]
return self._oauth2_token
def get_service_jwt(self):
if self._service_jwt is None:
if "service_credentials" not in self.config:
if "GOOGLE_APPLICATION_CREDENTIALS" in os.environ:
logger.info("Using GOOGLE_APPLICATION_CREDENTIALS file %s",
os.environ["GOOGLE_APPLICATION_CREDENTIALS"])
with open(os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
) as fh:
self.config.service_credentials = json.load(fh)
else:
raise NoServiceCredentials()
payload = {
'iss': self.config.service_credentials["client_email"],
'sub': self.config.service_credentials["client_email"],
'scope': self.scope,
'aud': "https://www.googleapis.com/oauth2/v4/token",
'iat': datetime.datetime.utcnow(),
'exp':
datetime.datetime.utcnow() + datetime.timedelta(minutes=60)
}
additional_headers = {
'kid': self.config.service_credentials["private_key_id"]
}
import jwt
self._service_jwt = jwt.encode(
payload,
self.config.service_credentials["private_key"],
headers=additional_headers,
algorithm='RS256').decode()
return self._service_jwt
def request(self, method, resource, **kwargs):
url = self.base_url + resource
res = self.get_session().request(method=method,
url=url,
timeout=self.timeout,
**kwargs)
res.raise_for_status()
return res if kwargs.get(
"stream") is True or method == "delete" else res.json()
def get(self, resource, **kwargs):
return self.request(method="get", resource=resource, **kwargs)
def post(self, resource, **kwargs):
return self.request(method="post", resource=resource, **kwargs)
def patch(self, resource, **kwargs):
return self.request(method="patch", resource=resource, **kwargs)
def put(self, resource, **kwargs):
return self.request(method="put", resource=resource, **kwargs)
def delete(self, resource, **kwargs):
return self.request(method="delete", resource=resource, **kwargs)
def get_project(self):
if "GOOGLE_CLOUD_PROJECT" in os.environ:
return os.environ["GOOGLE_CLOUD_PROJECT"]
self.get_session(
) # Ensures any available project-specific credentials are loaded.
if "service_credentials" in self.config:
return self.config.service_credentials["project_id"]
res = requests.get(self.project_id_metadata_url,
headers={"Metadata-Flavor": "Google"})
res.raise_for_status()
return res.content.decode()
def list(self, resource, include_prefixes=True, **kwargs):
while True:
page = self.request(method="get", resource=resource, **kwargs)
items = [dict(name=i) for i in page.get("prefixes", [])
] if include_prefixes else []
items.extend(page.get("items", []))
for item in items:
yield item
if "maxResults" in kwargs["params"]:
kwargs["params"]["maxResults"] -= 1
if kwargs["params"]["maxResults"] == 0:
return
if "nextPageToken" in page:
if not self.suppress_paging_warning:
logger.warn(
"Large number of results returned. Listing may take a while. "
"You can limit the object count using the --max-results option."
)
self.suppress_paging_warning = True
kwargs["params"]["pageToken"] = page["nextPageToken"]
else:
break
def get_presigned_url(self,
bucket,
key,
expires_at,
method="GET",
headers=None,
content_type=None,
md5_b64=""):
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization, hashes
from cryptography.hazmat.primitives.asymmetric import padding
string_to_sign = "\n".join(
[method, md5_b64, content_type or "",
str(int(expires_at))])
for header, value in (headers.items() if headers else {}):
string_to_sign += "\n" + header + ":" + value
string_to_sign += "\n/" + bucket + "/" + key
private_key_bytes = self.config.service_credentials[
"private_key"].encode()
private_key = serialization.load_pem_private_key(
private_key_bytes, password=None, backend=default_backend())
signature = private_key.sign(string_to_sign.encode(),
padding.PKCS1v15(), hashes.SHA256())
qs = dict(
GoogleAccessId=self.config.service_credentials["client_email"],
Expires=str(int(expires_at)),
Signature=base64.b64encode(signature).decode())
return self.presigned_url_base + bucket + "/" + key + "?" + requests.compat.urlencode(
qs)
def __init__(self, hub):
# If we're in development mode, rewrite some of our topics so that
# local playback with fedmsg-dg-replay works as expected.
if hub.config["environment"] == "dev":
# Keep the original set, but append a duplicate set for local work
prefix, env = hub.config["topic_prefix"], hub.config["environment"]
self.topic = self.topic + [
".".join([prefix, env] + topic.split(".")[3:]) for topic in self.topic
]
super(BugzillaTicketFiler, self).__init__(hub)
if not self._initialized:
return
# This is just convenient.
self.config = self.hub.config
# First, initialize fedmsg and bugzilla in this thread's context.
hostname = socket.gethostname().split(".", 1)[0]
if not getattr(getattr(fedmsg, "__local", None), "__context", None):
fedmsg.init(name="hotness.%s" % hostname)
fedmsg.meta.make_processors(**self.hub.config)
self.bugzilla = hotness.bz.Bugzilla(
consumer=self, config=self.config["hotness.bugzilla"]
)
self.buildsys = hotness.buildsys.Koji(
consumer=self, config=self.config["hotness.koji"]
)
default = "https://pagure.io/releng/fedora-scm-requests"
self.repo_url = self.config.get("hotness.repo_url", default)
default = "https://pdc.fedoraproject.org"
self.pdc_url = self.config.get("hotness.pdc_url", default)
default = "https://src.fedoraproject.org"
self.dist_git_url = self.config.get("hotness.dist_git_url", default)
anitya_config = self.config.get("hotness.anitya", {})
default = "https://release-monitoring.org"
self.anitya_url = anitya_config.get("url", default)
self.anitya_username = anitya_config.get("username", default)
self.anitya_password = anitya_config.get("password", default)
# Also, set up our global cache object.
_log.info("Configuring cache.")
with hotness.cache.cache_lock:
if not hotness.cache.cache.is_configured:
hotness.cache.cache.configure(**self.config['hotness.cache'])
self.mdapi_url = self.config.get("hotness.mdapi_url")
_log.info("Using hotness.mdapi_url=%r" % self.mdapi_url)
self.repoid = self.config.get("hotness.repoid", "rawhide")
_log.info("Using hotness.repoid=%r" % self.repoid)
self.distro = self.config.get("hotness.distro", "Fedora")
_log.info("Using hotness.distro=%r" % self.distro)
# Retrieve the requests configuration; by default requests time out
# after 15 seconds and are retried up to 3 times.
self.requests_session = requests.Session()
self.timeout = (
self.config.get("hotness.connect_timeout", 15),
self.config.get("hotness.read_timeout", 15),
)
retries = self.config.get("hotness.requests_retries", 3)
retry_conf = retry.Retry(
total=retries, connect=retries, read=retries, backoff_factor=1
)
retry_conf.BACKOFF_MAX = 5
self.requests_session.mount(
"http://", requests.adapters.HTTPAdapter(max_retries=retry_conf)
)
self.requests_session.mount(
"https://", requests.adapters.HTTPAdapter(max_retries=retry_conf)
)
_log.info(
"Requests timeouts are {}s (connect) and {}s (read)"
" with {} retries".format(self.timeout[0], self.timeout[1], retries)
)
# Build a little store where we'll keep track of what koji scratch
# builds we have kicked off. We'll look later for messages indicating
# that they have completed.
self.scratch_builds = {}
_log.info("That new hotness ticket filer is all initialized")
def __init__(self):
# This is just convenient.
self.config = conf["consumer_config"]
self.bugzilla = hotness.bz.Bugzilla(consumer=self,
config=self.config["bugzilla"])
self.buildsys = hotness.buildsys.Koji(consumer=self,
config=self.config["koji"])
default = "https://pdc.fedoraproject.org"
self.pdc_url = self.config.get("pdc_url", default)
default = "https://src.fedoraproject.org"
self.dist_git_url = self.config.get("dist_git_url", default)
anitya_config = self.config.get("anitya", {})
default = "https://release-monitoring.org"
self.anitya_url = anitya_config.get("url", default)
self.anitya_username = anitya_config.get("username", default)
self.anitya_password = anitya_config.get("password", default)
# Also, set up our global cache object.
_log.info("Configuring cache.")
with hotness.cache.cache_lock:
if not hotness.cache.cache.is_configured:
hotness.cache.cache.configure(**self.config["cache"])
self.mdapi_url = self.config.get("mdapi_url")
_log.info("Using hotness.mdapi_url=%r" % self.mdapi_url)
self.repoid = self.config.get("repoid", "rawhide")
_log.info("Using hotness.repoid=%r" % self.repoid)
self.distro = self.config.get("distro", "Fedora")
_log.info("Using hotness.distro=%r" % self.distro)
# Retrieve the requests configuration; by default requests time out
# after 15 seconds and are retried up to 3 times.
self.requests_session = requests.Session()
self.timeout = (
self.config.get("connect_timeout", 15),
self.config.get("read_timeout", 15),
)
retries = self.config.get("hotness.requests_retries", 3)
retry_conf = retry.Retry(total=retries,
connect=retries,
read=retries,
backoff_factor=1)
retry_conf.BACKOFF_MAX = 5
self.requests_session.mount(
"http://", requests.adapters.HTTPAdapter(max_retries=retry_conf))
self.requests_session.mount(
"https://", requests.adapters.HTTPAdapter(max_retries=retry_conf))
_log.info("Requests timeouts are {}s (connect) and {}s (read)"
" with {} retries".format(self.timeout[0], self.timeout[1],
retries))
# Build a little store where we'll keep track of what koji scratch
# builds we have kicked off. We'll look later for messages indicating
# that they have completed.
self.scratch_builds = {}
_log.info("That new hotness ticket filer is all initialized")
TESTCASE_CACHE = LRUCacheDict(max_size=1000, expiration=172800)
# The number of seconds to sleep after each test run to avoid DDOS.
SLEEP_TIME = 30
Testcase = collections.namedtuple('Testcase', ['id', 'job_type'])
# Configuring backoff retrying because sending a request to ClusterFuzz
# might fail during a deployment.
http = requests.Session()
http.mount(
'https://',
adapters.HTTPAdapter(
# backoff_factor is 0.5. Therefore, the max wait time is 16s.
retry.Retry(total=5,
backoff_factor=0.5,
status_forcelist=[500, 502, 503, 504])))
def post(*args, **kwargs): # pragma: no cover
"""Make a post request. This method is needed for mocking."""
return http.post(*args, **kwargs)
def load_sanity_check_testcase_ids():
"""Return a list of all testcases to try."""
with open(SANITY_CHECKS) as stream:
return yaml.load(stream)['testcase_ids']
def build_command(args):
DEFAULT_CONNECTION_TIMEOUT = 1
DEFAULT_READ_TIMEOUT = 3
DEFAULT_CONNECTION_RETRIES = 1
DEFAULT_READ_RETRIES = 1
DEFAULT_TOTAL_RETRIES = 1
DEFAULT_STATUS_CODES_TO_RETRY_ON = range(500, 600)
Timeout = collections.namedtuple('Timeout',
['connection_timeout', 'read_timeout'])
DEFAULT_TIMEOUT = Timeout(connection_timeout=DEFAULT_CONNECTION_TIMEOUT,
read_timeout=DEFAULT_READ_TIMEOUT)
DEFAULT_RETRY = retry.Retry(
total=DEFAULT_TOTAL_RETRIES,
connect=DEFAULT_CONNECTION_RETRIES,
read=DEFAULT_READ_RETRIES,
status_forcelist=DEFAULT_STATUS_CODES_TO_RETRY_ON,
)
class ServiceCaller:
"""
A class for calling :mod:`services <apiron.service.base>`
"""
@staticmethod
def get_adapted_session(adapter):
"""
Mounts an adapter capable of communication over HTTP or HTTPS to the supplied session.
:param adapter:
A :class:`requests.adapters.HTTPAdapter` instance
This module provides utilities for making HTTP calls using the requests library.
"""
from google.auth import transport
import requests
from requests.packages.urllib3.util import retry # pylint: disable=import-error
_ANY_METHOD = None
# Default retry configuration: Retries once on low-level connection and socket read errors.
# Retries up to 4 times on HTTP 500 and 503 errors, with exponential backoff. Returns the
# last response upon exhausting all retries.
DEFAULT_RETRY_CONFIG = retry.Retry(connect=1,
read=1,
status_forcelist=[500, 503],
method_whitelist=_ANY_METHOD,
raise_on_status=False,
backoff_factor=0.5)
class HttpClient:
"""Base HTTP client used to make HTTP calls.
HttpClient maintains an HTTP session, and handles request authentication and retries if
necessary.
"""
def __init__(self,
credential=None,
session=None,
base_url='',
headers=None,
"""Internal HTTP client module.
This module provides utilities for making HTTP calls using the requests library.
"""
from google.auth import transport
import requests
from requests.packages.urllib3.util import retry # pylint: disable=import-error
# Default retry configuration: Retries once on low-level connection and socket read errors.
# Retries up to 4 times on HTTP 500 and 503 errors, with exponential backoff. Returns the
# last response upon exhausting all retries.
DEFAULT_RETRY_CONFIG = retry.Retry(
connect=1, read=1, status=4, status_forcelist=[500, 503],
raise_on_status=False, backoff_factor=0.5)
class HttpClient(object):
"""Base HTTP client used to make HTTP calls.
HttpClient maintains an HTTP session, and handles request authentication and retries if
necessary.
"""
def __init__(
self, credential=None, session=None, base_url='', headers=None,
retries=DEFAULT_RETRY_CONFIG):
"""Creates a new HttpClient instance from the provided arguments.
def __init__(self, hub):
# If we're in development mode, rewrite some of our topics so that
# local playback with fedmsg-dg-replay works as expected.
if hub.config['environment'] == 'dev':
# Keep the original set, but append a duplicate set for local work
prefix, env = hub.config['topic_prefix'], hub.config['environment']
self.topic = self.topic + [
'.'.join([prefix, env] + topic.split('.')[3:])
for topic in self.topic
]
super(BugzillaTicketFiler, self).__init__(hub)
if not self._initialized:
return
# This is just convenient.
self.config = self.hub.config
# First, initialize fedmsg and bugzilla in this thread's context.
hostname = socket.gethostname().split('.', 1)[0]
fedmsg.init(name='hotness.%s' % hostname)
fedmsg.meta.make_processors(**self.hub.config)
self.bugzilla = hotness.bz.Bugzilla(
consumer=self, config=self.config['hotness.bugzilla'])
self.buildsys = hotness.buildsys.Koji(
consumer=self, config=self.config['hotness.koji'])
default = 'https://admin.fedoraproject.org/pkgdb/api'
self.pkgdb_url = self.config.get('hotness.pkgdb_url', default)
anitya_config = self.config.get('hotness.anitya', {})
default = 'https://release-monitoring.org'
self.anitya_url = anitya_config.get('url', default)
self.anitya_username = anitya_config.get('username', default)
self.anitya_password = anitya_config.get('password', default)
# Also, set up our global cache object.
_log.info("Configuring cache.")
with hotness.cache.cache_lock:
if not hasattr(hotness.cache.cache, 'backend'):
hotness.cache.cache.configure(**self.config['hotness.cache'])
self.mdapi_url = self.config.get("hotness.mdapi_url")
_log.info("Using hotness.mdapi_url=%r" % self.mdapi_url)
self.repoid = self.config.get('hotness.repoid', 'rawhide')
_log.info("Using hotness.repoid=%r" % self.repoid)
self.distro = self.config.get('hotness.distro', 'Fedora')
_log.info("Using hotness.distro=%r" % self.distro)
# Retrieve the requests configuration; by default requests time out
# after 15 seconds and are retried up to 3 times.
self.requests_session = requests.Session()
self.timeout = (
self.config.get('hotness.connect_timeout', 15),
self.config.get('hotness.read_timeout', 15),
)
retries = self.config.get('hotness.requests_retries', 3)
retry_conf = retry.Retry(total=retries,
connect=retries,
read=retries,
backoff_factor=1)
retry_conf.BACKOFF_MAX = 5
self.requests_session.mount(
'http://', requests.adapters.HTTPAdapter(max_retries=retry_conf))
self.requests_session.mount(
'https://', requests.adapters.HTTPAdapter(max_retries=retry_conf))
_log.info('Requests timeouts are {}s (connect) and {}s (read)'
' with {} retries'.format(self.timeout[0], self.timeout[1],
retries))
# Build a little store where we'll keep track of what koji scratch
# builds we have kicked off. We'll look later for messages indicating
# that they have completed.
self.scratch_builds = {}
_log.info("That new hotness ticket filer is all initialized")
