def __init__(self, url, timeout):
self.status_code = 200
self.text = 200
number_of_attempts.append(1)
if url not in urls_to_resolve:
urls_to_resolve.append(url)
if url in [ORIGINAL_V1_URL, ORIGINAL_V2_URL, INVALID_V1_URL]:
raise requests.URLRequired('invalid url: {0}'.format(url))
elif url == ILLEGAL_URL:
raise requests.URLRequired(
'unknown url type: {0}'.format(url))
elif url in [VALID_V1_URL, VALID_V2_URL]:
return None
elif url == TIMEOUT_URL:
raise requests.ConnectionError(
'Timeout while trying to import')
elif url == BAD_RESPONSE_CODE_URL:
self.status_code = 404
self.text = 404
elif url == RETRY_URL:
if len(number_of_attempts) < MAX_NUMBER_RETRIES:
raise requests.ConnectionError(
'Timeout while trying to import')
else:
return None
def test_update_notable_invalid_url(self, mocked_requests_post):
""" Test update notable with wrong URL or connection issue"""
print("Test update_notable returns HTTPError\n")
try:
sim_status = 1
sim_content = '<response><sessionKey>' + self.simSessionKey + '</sessionKey></response>'
mocked_requests_post.return_value = self._generateResponse(
sim_content, 200)
splnk_utils = splunk_utils.SplunkUtils(host=self.fake_host,
port=self.fake_port,
username=self.fake_username,
password=self.fake_password,
verify=self.verify)
mocked_requests_post.side_effect = requests.URLRequired(
Mock(status=404), "Ambiguous excetpion.")
ret = splnk_utils.update_notable(event_id=self.simEventId,
comment=self.simComment,
status=sim_status,
cafile=self.verify)
#
# request post throws exception
#
assert False
except splunk_utils.RequestError as e:
assert True
def setup(self, api_host=None, api_version=None, api_key=None, api_secret=None,
username=None, password=None,
access_token=None, token_issuer_path=None,
token_issuer_host=None, **kwargs):
self.api_version = api_version or environ.get('TRUSTPILOT_API_VERSION', 'v1')
self.api_host = api_host or environ.get('TRUSTPILOT_API_HOST', 'https://api.trustpilot.com')
self.token_issuer_host = token_issuer_host or self.api_host
self.access_token = access_token
self.token_issuer_path = token_issuer_path or environ.get(
'TRUSTPILOT_API_TOKEN_ISSUER_PATH', "oauth/oauth-business-users-for-applications/accesstoken")
self.hooks = dict()
if not self.api_host.startswith("http"):
raise requests.URLRequired(
"'{}' is not a valid api_host url".format(api_host))
try:
self.api_key=api_key or environ['TRUSTPILOT_API_KEY']
self.api_secret=api_secret or environ.get('TRUSTPILOT_API_SECRET', '')
self.username=username or environ.get('TRUSTPILOT_USERNAME')
self.password=password or environ.get('TRUSTPILOT_PASSWORD')
self.access_token=access_token
self.hooks['response'] = self._post_request_callback
except KeyError as e:
logger.debug("Not auth setup, missing env-var or setup for {}".format(e))
return self
def make_request(self, mapped_data, config):
request_type = config['type']
url = config['url']
try:
full_url = url.format(**mapped_data)
except KeyError as ker:
LOG.error(f'Error sending message in job {self._id}: {ker}' +
f'{url} -> {mapped_data}')
raise requests.URLRequired(f'bad argument in URL: {ker}')
fn = self.get_request_function_for_type(request_type)
auth = config.get('basic_auth')
if auth:
auth = HTTPBasicAuth(auth['user'], auth['password'])
params = config.get('query_params')
if params:
params = self.data_from_datamap(mapped_data, params)
json_body = config.get('json_body')
if json_body:
json_body = self.data_from_datamap(mapped_data, json_body)
headers = config.get('token')
if headers:
headers = {'Authorization': f'access_token {headers}'}
LOG.debug(f'json: {json_body}, url: {full_url}')
return fn(full_url,
auth=auth,
headers=headers,
params=params,
json=json_body)
def test_url_required_exception(self):
with mock.patch('requests.get') as requests_get_mock:
requests_get_mock.side_effect = requests.URLRequired()
with StdoutCapture() as output:
self.api_caller.get_resource("/resource", DiscoveryContainer())
self.assertIn("ERROR::Get url=http://1.2.3.4:567/resource Error <class "
"'requests.exceptions.URLRequired'>:",
output.raw)
def test_add_threat_intel_item_errors(self, mocked_requests_post):
# 1. Connect successfully
sim_content = '<response><sessionKey>' + self.simSessionKey + '</sessionKey></response>'
mocked_requests_post.return_value = self._generateResponse(
sim_content, 200)
splnk_utils = splunk_utils.SplunkUtils(host=self.fake_host,
port=self.fake_port,
username=self.fake_username,
password=self.fake_password,
verify=self.verify)
# 2. Simulate wrong intel type
try:
splnk_utils.add_threat_intel_item("Fake type", {}, False)
except splunk_utils.RequestError as e:
print("Fake intel type causes exception as expected.")
assert (True)
# 3. Simulate RequestException
threat_type = "ip_intel"
mocked_requests_post.side_effect = requests.RequestException(
Mock(status=404), "Ambiguous excetpion.")
try:
splnk_utils.add_threat_intel_item(threat_type, {}, False)
except splunk_utils.RequestError:
assert True
# 4. Simulate ConnectionError
mocked_requests_post.side_effect = requests.ConnectionError(
Mock(status=404), "Ambiguous excetpion.")
try:
splnk_utils.add_threat_intel_item(threat_type, {}, False)
except splunk_utils.RequestError:
assert True
# 5. Simulate HttpError
mocked_requests_post.side_effect = requests.HTTPError(
Mock(status=404), "Ambiguous excetpion.")
try:
splnk_utils.add_threat_intel_item(threat_type, {}, False)
except splunk_utils.RequestError:
assert True
# 6. Simulate URLRequired
mocked_requests_post.side_effect = requests.URLRequired(
Mock(status=404), "Ambiguous excetpion.")
try:
splnk_utils.add_threat_intel_item(threat_type, {}, False)
except splunk_utils.RequestError:
assert True
# 7. Simulate TooManyRedirects
mocked_requests_post.side_effect = requests.TooManyRedirects(
Mock(status=404), "Ambiguous excetpion.")
try:
splnk_utils.add_threat_intel_item(threat_type, {}, False)
except splunk_utils.RequestError:
assert True
def do_request(_id, config, mapped_data, override_url=None):
url = config['url']
try:
if not override_url:
full_url = url.format(**mapped_data)
else:
full_url = override_url
except KeyError as ker:
# self comes from parent class?
LOG.error(f'Error sending message in job {_id}: {ker}' +
f'{url} -> {mapped_data}')
raise requests.URLRequired(f'bad argument in URL: {ker}')
fn = REST_CALLS[config['type'].upper()]
auth = config.get('basic_auth')
if auth:
auth = HTTPBasicAuth(auth['user'], auth['password'])
params = config.get('query_params')
if params and not override_url:
params = data_from_datamap(mapped_data, params)
else:
params = None
form_body = config.get('form_body')
if form_body:
form_body = data_from_datamap(mapped_data, config.get('form_body'))
json_body = config.get('json_body')
if json_body:
json_body = data_from_datamap(mapped_data, json_body)
token = config.get('token', {})
if token:
token = {'Authorization': f'access_token {token}'}
headers = config.get('headers', {})
if headers:
headers = data_from_datamap(mapped_data, headers)
headers = {**token, **headers} # merge in token if we have one
request_kwargs = {
'auth': auth,
'headers': headers,
'params': params,
'json': json_body,
'data': form_body,
'verify': False
}
if not config.get('mock_request', False):
return fn(
full_url,
**request_kwargs
)
request_kwargs['full_url'] = full_url
LOG.debug(request_kwargs)
try:
ERR.report(_id, json.dumps(request_kwargs))
except TypeError:
ERR.report(_id, json.dumps(
[(k, str(v)) for k, v in request_kwargs.items()]
))
raise ReportableError()
def test_invoke_elem_urlerror(self):
"""Tests handling of URLError"""
na_element = fake.FAKE_NA_ELEMENT
self.mock_object(self.root, '_create_request',
mock.Mock(return_value=fake.FAKE_NA_ELEMENT))
self.mock_object(api, 'LOG')
self.root._session = fake.FAKE_HTTP_SESSION
self.mock_object(self.root, '_build_session')
self.mock_object(self.root._session, 'post',
mock.Mock(side_effect=requests.URLRequired()))
self.assertRaises(exception.StorageCommunicationException,
self.root.invoke_elem, na_element)
def test_invoke_elem_urlerror(self):
"""Tests handling of URLError"""
na_element = fake.FAKE_NA_ELEMENT
element_name = fake.FAKE_NA_ELEMENT.get_name()
self._setup_mocks_for_invoke_element(
mock_post_action=mock.Mock(side_effect=requests.URLRequired()))
self.assertRaises(exception.StorageCommunicationException,
self.root.invoke_elem,
na_element)
self.assertTrue(self.root._get_base_url.called)
self.root._get_request_info.assert_called_once_with(
element_name, self.root._session)
def _discovery(prefix=""):
if JENKINS_URL == "":
raise requests.URLRequired("ERROR: Url required")
jobs = SESSION.get(JENKINS_URL + '/view/All/api/json')
if jobs.status_code == requests.codes.ok:
data = {'data': []}
if prefix.lower() == "":
for job in jobs.json().get('jobs'):
if job.get('color') != "disabled":
r = SESSION.get(JENKINS_URL + "/job/" + job.get('name') +
"/api/json")
if r.status_code == requests.codes.ok:
if r.json().get('jobs'):
for branch in r.json().get('jobs'):
result = {
'{#JOBNAME}': job.get('name'),
'{#BRANCHNAME}': branch.get('name')
}
data['data'].append(result)
else:
result = {'{#JOBNAME}': job.get('name')}
data['data'].append(result)
elif prefix is not None:
for job in jobs.json().get('jobs'):
if job.get('name').upper() == prefix.upper(
) and job.get('color') != "disabled":
r = SESSION.get(JENKINS_URL + "/job/" + job.get('name') +
"/api/json")
if r.status_code == requests.codes.ok:
if r.json().get('jobs'):
for branch in r.json().get('jobs'):
result = {
'{#JOBNAME}': job.get('name'),
'{#BRANCHNAME}': branch.get('name')
}
data['data'].append(result)
else:
result = {'{#JOBNAME}': job.get('name')}
data['data'].append(result)
return json.dumps(data)
else:
jobs.raise_for_status()
def _rest(job_name="", branch_name="", max_time=0):
if JENKINS_URL == "":
raise requests.URLRequired("ERROR: Url required")
url = JENKINS_URL + '/job/' + job_name
if branch_name != "":
url = url + "/job/" + branch_name
url = url + '/lastBuild/api/json'
r = SESSION.get(url, verify=False)
if r.status_code == 200:
try:
job = r.json()
if (job.get('building') is False) and (job.get('result')
== 'SUCCESS'):
return 1
# check if job exceeded the estimated duration
if (max_time > 0) and (job.get('building') is
True) and (job.get('result') is None):
current = 0
max_estimated = job.get('estimatedDuration')
started = str(job.get('timestamp'))
tmp = dt.fromtimestamp(float(started[:10]))
duration = (dt.now() - tmp).seconds * 1000
if duration > 0:
current = (duration * 100) / max_estimated
if current > max_time:
# this is when job exceeds its estimated time
return -2
if (job.get('building') is True) and (job.get('result') is None):
# job is running
return 0
# job failed
return -1
except Exception as e:
sys.stderr.write("Failed %s" % e)
pass
# job failure
return -1
stat = sub.add_parser('status')
stat.add_argument('name', nargs='?', default="")
stat.add_argument('branch', nargs='?', default="")
stat.add_argument('max_time', nargs='?', default=0)
stat.set_defaults(func=status)
args = parser.parse_args()
try:
SESSION = requests.session()
Path(args.config).resolve(strict=True)
config = ConfigObj(args.config)
HOSTNAME = config.get('hostname')
USERNAME = config.get('username')
PASSWORD = config.get('password')
JENKINS_URL = config.get('jenkins_url')
PREFIX = config.get('prefix', "")
SESSION.auth = (USERNAME, PASSWORD)
# Lazy validation :-)
if HOSTNAME == "":
raise requests.URLRequired("ERROR: Hostname Required!")
if JENKINS_URL == "":
raise requests.URLRequired("ERROR: Jenkins Url Required!")
args.func(args)
except Exception as E:
print(E)
sys.exit(1)
