def rest(self,
endpoint,
data=None,
method='get',
formatjson=True,
params=None):
url = self.protocol + "://" + self.host + ":" + str(
self.port) + "/api/atlas/" + endpoint
header = {
"Accept": "application/json",
"Content-Type": "application/json"
}
if kerberos == True:
auth = HTTPKerberosAuth()
else:
auth = (self.username, self.password)
try:
r = requests.request(method,
url,
headers=header,
auth=auth,
verify=False,
data=data,
params=params)
except:
logger.error("Cannot connect to Atlas")
return (False)
if formatjson:
return (json.loads(r.text))
else:
return (r.text)
def get(params):
url = DATAGREPPER_URL
headers = {'Accept': 'application/json', }
response = requests.get(url=url, params=params, headers=headers,
auth=HTTPKerberosAuth(mutual_authentication=OPTIONAL))
return response.json()
def _krb_auth(self):
retcode = subprocess.Popen(["klist", "-s"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).wait()
if retcode or self.ktfile:
if self.ktfile:
retcode = subprocess.Popen(
[
"kinit",
self.krb_princ,
"-k",
"-t",
self.ktfile,
"-c",
self.ccache_file,
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
).wait()
os.environ["KRB5CCNAME"] = self.ccache_file
else:
# If keytab path wasn't provided, default location will be attempted
retcode = subprocess.Popen(
["kinit", self.krb_princ, "-k", "-c", self.ccache_file],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
).wait()
os.environ["KRB5CCNAME"] = self.ccache_file
# preemptive auth is forced to speed up parallel requests
return HTTPKerberosAuth(
mutual_authentication=OPTIONAL,
force_preemptive=True,
)
def hdfs_client_ini(conf):
_conf = conf
_url = ''
_nodes = []
for _node in _conf['namenodes']:
_nodes.append('http://' + str(_node) + ':' + str(_conf['port']))
_url = ';'.join(_nodes)
if os.path.isfile(_conf['keytab']):
_conf_keytab = _conf['keytab']
else:
_conf_keytab = str(os.path.dirname(
os.path.realpath(__file__))) + os.sep + str(_conf['keytab'])
try:
os.environ["KRB5_CLIENT_KTNAME"] = _conf_keytab
except Exception as _err:
print('ERR: [initiator:hdfs_client_ini]', _err)
return False
try:
_kerberos_auth = HTTPKerberosAuth(principal=_conf['principal'])
except Exception as _err:
print('ERR: [initiator:hdfs_client_ini]', _err)
return False
else:
try:
_client = KerberosClient(_url)
except Exception as _err:
print('ERR: [initiator:hdfs_client_ini]', _err)
return False
else:
return _client
def set_kerberos_auth(self):
"""Set up kerberos auth for the client, based on the current ticket."""
mutual_auth = conf.KERBEROS.MUTUAL_AUTHENTICATION.get().upper()
if mutual_auth == 'OPTIONAL':
self._session.auth = HTTPKerberosAuth(
mutual_authentication=OPTIONAL)
elif mutual_auth == 'REQUIRED':
self._session.auth = HTTPKerberosAuth(
mutual_authentication=REQUIRED)
elif mutual_auth == 'DISABLED':
self._session.auth = HTTPKerberosAuth(
mutual_authentication=DISABLED)
else:
self._session.auth = HTTPKerberosAuth(
mutual_authentication=OPTIONAL)
return self
def get_brew_builds(errata_id, session=None):
"""5.2.2.1. GET /api/v1/erratum/{id}/builds
Get Errata list of builds.
https://errata.devel.redhat.com/developer-guide/api-http-api.html#api-get-apiv1erratumidbuilds
:param str errata_id: the errata id
:param requests.Session session: A python-requests Session object,
used for for connection pooling. Providing `session` object can
yield a significant reduction in total query time when looking up
many builds.
http://docs.python-requests.org/en/master/user/advanced/#session-objects
:return: A List of initialized Build object with the build details
:raises exceptions.BrewBuildException: When erratum return errors
"""
if session is None:
session = requests.session()
res = session.get(constants.errata_get_builds_url.format(id=errata_id),
verify=ssl.get_default_verify_paths().openssl_cafile,
auth=HTTPKerberosAuth())
brew_list = []
if res.status_code == 200:
jlist = res.json()
for key in jlist.keys():
for obj in jlist[key]['builds']:
brew_list.append(
brew.Build(nvr=list(obj.keys())[0], product_version=key))
return brew_list
else:
raise exceptions.BrewBuildException(
"fetch builds from {id}: {msg}".format(id=errata_id, msg=res.text))
def __init__(self, endpoint, headers, retry_policy):
self._endpoint = endpoint
self._headers = headers
self._retry_policy = retry_policy
if self._endpoint.auth == constants.AUTH_KERBEROS:
self._auth = HTTPKerberosAuth(**conf.kerberos_auth_configuration())
elif self._endpoint.auth == constants.AUTH_BASIC:
self._auth = (self._endpoint.username, self._endpoint.password)
elif self._endpoint.auth == constants.GOOGLE_AUTH:
credentials = (conf.google_auth_credentials())
self._auth = credentials
#Once we have credentials, we attach them to a transport. We use the transport to
#make authenticated requests: how does self._session work? Do I not have to use
#AuthorizedSession? Kerberos HTTPKerberosAuth attaches Kerberos Auth to Requests object
#so should self._auth = credentials or authed_session?
authed_session = AuthorizedSession(credentials)
elif self._endpoint.auth != constants.NO_AUTH:
raise BadUserConfigurationException(u"Unsupported auth %s" %
self._endpoint.auth)
self._session = requests.Session()
self.logger = SparkLog(u"ReliableHttpClient")
self.verify_ssl = not conf.ignore_ssl_errors()
if not self.verify_ssl:
self.logger.debug(
u"ATTENTION: Will ignore SSL errors. This might render you vulnerable to attacks."
)
requests.packages.urllib3.disable_warnings()
def put_request(self, suffix_url, xml_file):
"""Make a oozie put request"""
status = False
file_h = open(os.path.join(self.cfg_mgr.saves, xml_file))
data = file_h.read()
http_headers = {'Content-Type': 'application/xml'}
url = self.oozie_url + suffix_url
self.logger.info(url)
response = requests.post(url,
data=data,
headers=http_headers,
auth=HTTPKerberosAuth())
if 200 <= response.status_code and response.status_code < 300:
my_json = json.dumps(response.json())
res_val = response.json()
# self.logger.info(json.dumps(response.json(),
# indent=4, sort_keys=True))
msg = "\n\n\t\tGo here: https://{0}:8888/oozie/" \
"list_oozie_workflow/{1}/\n".\
format(self.cfg_mgr.workflow_host, res_val['id'])
self.logger.info(msg)
status = True
else:
self.logger.error("Error submitting job. Error code: "
"{status}".format(status=response.status_code))
self.logger.error(response.text)
return status
def advisory_drop_cli(advisory):
"""Drop advisory
Advisories can only be dropped by the creators, and program management.
This script can get run on buildvm with the credentials of the creator of
ART's advisories, so the bot account can drop them.
"""
url = errata_drop_url.format(id=advisory)
data = 'utf8=%E2%9C%93&reason=Dropping+unused+advisory%21&commit=Dropping+unused+advisory'
headers = {"Content-Type": "text/plain"}
r = requests.post(url, auth=HTTPKerberosAuth(), data=data, headers=headers)
if r.status_code == 200:
click.echo(f'Succesfully dropped advisory {advisory}')
sys.exit(0)
elif "ERROR: Validation failed: Previous - Transition DROPPED_NO_SHIP => DROPPED_NO_SHIP is invalid" in r.text:
click.echo(f'Advisory {advisory} already seems dropped')
sys.exit(0)
else:
click.echo(
f'Failed to drop advisory {advisory}. Got status code {r.status_code}'
)
click.echo(r.text)
sys.exit(1)
def main():
with get(f"https://rhevm.abc.idm.lab.eng.brq.redhat.com/ovirt-engine/api/vms/{config['machine_id']}/statistics",
auth=HTTPKerberosAuth(), headers={'accept': 'application/json'}) as response:
if response.ok:
result = loads(response.text)
else:
return False
def get_comments(advisory_id):
"""5.2.10.2. GET /api/v1/comments?filter[key]=value
Retrieve all advisory comments
Example request body:
{"filter": {"errata_id": 11112, "type": "AutomatedComment"}}
Returns an array of comments ordered in descending order
(newest first). The array may be empty depending on the filters
used. The meaning of each attribute is documented under GET
/api/v1/comments/{id} (see Erratum.get_comment())
Included for reference:
5.2.10.2.1. Filtering
The list of comments can be filtered by applying
filter[key]=value as a query parameter. All attributes of a
comment - except advisory_state - can be used as a filter.
This is a paginated API. Reference documentation:
https://errata.devel.redhat.com/developer-guide/api-http-api.html#api-pagination
"""
body = {"filter": {"errata_id": advisory_id, "type": "Comment"}}
res = requests.get(constants.errata_get_comments_url,
verify=ssl.get_default_verify_paths().openssl_cafile,
auth=HTTPKerberosAuth(),
json=body)
if res.status_code == 200:
return res.json().get('data', [])
elif res.status_code == 401:
raise exceptions.ErrataToolUnauthorizedException(res.text)
else:
return False
def mentioned_trend(baseurl,mysqlhostIP, mysqlUserName = '******', mysqlPassword = '', dbname = 'btv_v2'):
list_key_words = list()
# 存储评论数据
# 连接数据库
print(base64.b64decode(b'Q29weXJpZ2h0IChjKSAyMDEyIERvdWN1YmUgSW5jLiBBbGwgcmlnaHRzIHJlc2VydmVkLg==').decode())
sqlConn=MySQLdb.connect(host=mysqlhostIP, user=mysqlUserName, passwd=mysqlPassword, db = dbname, charset='utf8')
sqlcursor = sqlConn.cursor()
sqlcursor.execute('''CREATE TABLE IF NOT EXISTS key_kk(pk bigint NOT NULL PRIMARY KEY AUTO_INCREMENT, keywords varchar(50)) DEFAULT CHARSET=utf8;''')
print '新建库成功'
os.popen('kinit -k -t /home/ctvit/ctvit.keytab ctvit')
kerberos_auth = HTTPKerberosAuth(mutual_authentication=OPTIONAL)
tablename = "DATA:WEIBO_POST_Keywords"
r = requests.get(baseurl + "/" + tablename + "/*", auth=kerberos_auth, headers = {"Accept" : "application/json"})
if issuccessful(r) == False:
print "Could not get messages from HBase. Text was:\n" + r.text
# quit()
bleats = json.loads(r.text)
box = list()
for row in bleats['Row']:
# print 000
# count+=1
message = ''
lineNumber = ''
username = ''
for cell in row['Cell']:
columnname = base64.b64decode(cell['column'])
value = cell['$']
if value == None:
print 'none'
continue
if columnname == "base_info:match":
key_word = base64.b64decode(value)
if columnname == "base_info:hot":
hot = base64.b64decode(value)
print 'hot',hot
# if ("北京卫视春晚" not in key_word) and ("北京台的春晚" not in key_word) and ("BTV春晚" not in key_word) and ("BTV春晚" not in key_word) and ("bTV春晚" not in key_word):
# break
# if columnname == "base_info:cdate":
# cdate = base64.b64decode(value)
# cdate = cdate.split('T')[0]
# print 'date',cdate
# if cdate not in box:
# box.append(cdate)
# for i in box:
# print i
# print 'ppp',type(key_word)
# print '11',key_word
# if key_word not in list_key_words:
# list_key_words.append(key_word)
#
# tempData = []
# for i in list_key_words:
# print 'key',i
# tempData.append(str(i))
# sqlcursor.execute('''insert into key_kk(keywords) values (%s)''',tempData)
# sqlConn.commit()
# tempData = []
sqlConn.close()
def __init__(self):
self.url = Environment().get_spark_livy_url()
self.auth = None
if Environment().is_kerberos_enabled():
from requests_kerberos import HTTPKerberosAuth, REQUIRED
self.auth = HTTPKerberosAuth(mutual_authentication=REQUIRED,
sanitize_mutual_error_response=False)
def atlasPOST(restAPI, data):
kerberos_auth = HTTPKerberosAuth(mutual_authentication=OPTIONAL)
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
url = "https://" + ATLAS_DOMAIN + restAPI
print(url)
r = requests.post(url, auth=kerberos_auth, verify=False, json=data)
return (json.loads(r.text))
def __init__(self):
if RAZ.API_AUTHENTICATION.get() == 'kerberos':
auth = HTTPKerberosAuth()
else:
auth = None
self.ranger = RangerRazAdls(RAZ.API_URL.get(), auth)
def _handle_idbroker_ha(fs=None):
fs = validate_fs(fs)
idbrokeraddr = get_conf().get(_CNF_CAB_ADDRESS % fs) if fs else None
response = None
if fs:
id_broker_addr = get_conf().get(_CNF_CAB_ADDRESS % fs)
if id_broker_addr:
id_broker_addr_list = id_broker_addr.split(',')
for id_broker_addr in id_broker_addr_list:
try:
response = requests.get(id_broker_addr +
'dt/knoxtoken/api/v1/token',
auth=HTTPKerberosAuth(),
verify=False)
except Exception as e:
if 'Name or service not known' in str(e):
LOG.warn('IDBroker %s is not available for use' %
id_broker_addr)
# Check response for None and if response code is successful (200) or authentication needed (401)
if (response is not None) and (response.status_code
in (200, 401)):
idbrokeraddr = id_broker_addr
break
return idbrokeraddr
else:
return idbrokeraddr
else:
return idbrokeraddr
def __get_es_client(self):
if self.auth_type == ESLoggerHandler.AuthType.NO_AUTH:
if self._client is None:
self._client = Elasticsearch(hosts=self.hosts,
use_ssl=self.use_ssl,
verify_certs=self.verify_certs,
connection_class=RequestsHttpConnection,
serializer=self.serializer)
return self._client
if self.auth_type == ESLoggerHandler.AuthType.BASIC_AUTH:
if self._client is None:
return Elasticsearch(hosts=self.hosts,
http_auth=self.auth_details,
use_ssl=self.use_ssl,
verify_certs=self.verify_certs,
connection_class=RequestsHttpConnection,
serializer=self.serializer)
return self._client
if self.auth_type == ESLoggerHandler.AuthType.KERBEROS_AUTH:
if not KERBEROS_SUPPORTED:
raise EnvironmentError("Kerberos module not available. Please install \"requests-kerberos\"")
# For kerberos we return a new client each time to make sure the tokens are up to date
return Elasticsearch(hosts=self.hosts,
use_ssl=self.use_ssl,
verify_certs=self.verify_certs,
connection_class=RequestsHttpConnection,
http_auth=HTTPKerberosAuth(mutual_authentication=DISABLED),
serializer=self.serializer)
raise ValueError("Authentication method not supported")
def krb_sign_on(url, cookiejar=None):
"""
Perform Kerberos-backed single-sign on against a provided
(protected) URL.
It is assumed that the current session has a working Kerberos
ticket.
Returns a Requests `CookieJar`, which can be accessed as a
dictionary, but most importantly passed directly into a request or
session via the `cookies` keyword argument.
If a cookiejar-like object (such as a dictionary) is passed as the
cookiejar keword argument, this is passed on to the Session.
"""
kerberos_auth = HTTPKerberosAuth(mutual_authentication=OPTIONAL)
with requests.Session() as s:
krb_auth_url = _init_session(s=s,
url=url,
cookiejar=cookiejar,
auth_url_fragment=u"auth/integrated/")
# Perform actual Kerberos authentication
log.info("Performing Kerberos authentication against %s" %
krb_auth_url)
r2 = s.get(krb_auth_url,
auth=kerberos_auth,
timeout=DEFAULT_TIMEOUT_SECONDS)
return _finalise_login(s, auth_results=r2)
def get_brew_build(nvr, product_version='', session=None):
"""5.2.2.1. GET /api/v1/build/{id_or_nvr}
Get Brew build details.
https://errata.devel.redhat.com/developer-guide/api-http-api.html#api-get-apiv1buildid_or_nvr
:param str nvr: A name-version-release string of a brew rpm/image build
:param str product_version: The product version tag as given to ET
when attaching a build
:param requests.Session session: A python-requests Session object,
used for for connection pooling. Providing `session` object can
yield a significant reduction in total query time when looking up
many builds.
http://docs.python-requests.org/en/master/user/advanced/#session-objects
:return: An initialized Build object with the build details
:raises exceptions.BrewBuildException: When build not found
"""
if session is None:
session = requests.session()
res = session.get(constants.errata_get_build_url.format(id=nvr),
verify=ssl.get_default_verify_paths().openssl_cafile,
auth=HTTPKerberosAuth())
if res.status_code == 200:
return brew.Build(nvr=nvr,
body=res.json(),
product_version=product_version)
else:
raise exceptions.BrewBuildException("{build}: {msg}".format(
build=nvr, msg=res.text))
def _do_request(self, requests_method, request_url, body, params, headers,
expected_status_code, stream, verify, timeout):
auth = HTTPKerberosAuth() if self.has_kerberos() else None
response = requests_method(request_url,
data=body,
params=params,
headers=headers,
stream=stream,
verify=verify,
timeout=timeout or self.default_timeout_sec,
auth=auth)
if self.logger.isEnabledFor(logging.DEBUG):
for hdr, hdr_content in list(response.request.headers.items()):
self.logger.debug('request header: %s: %s' %
(hdr, hdr_content))
self.logger.debug(
'reply: "%s %s" %s' %
(response.status_code, response.reason, response.content))
for hdr, hdr_content in list(response.headers.items()):
self.logger.debug('response header: %s: %s' %
(hdr, hdr_content))
if response.status_code != expected_status_code:
self._raise_client_error(response, request_url)
if stream:
return StreamedResponse(response)
response_json = response.json()
if response.history:
response_json['history'] = response.history
return response_json
def _create_requests_session(self):
"""
Creates a Requests Session and authenticates to base API URL with kerberos-requests.
We're using a Session to persist cookies across all requests made from the Session instance.
:return s: Requests Session.
"""
# TODO: Support other authentication methods.
# Set up authentication for requests session.
s = requests.Session()
if self.auth == 'kerberos':
self.principal = _get_kerberos_principal()
s.auth = HTTPKerberosAuth(mutual_authentication=DISABLED)
s.verify = False
if isinstance(self.auth, tuple):
s.auth = self.auth
# Try to authenticate to auth_url.
try:
r = s.get(self.auth_url)
r.raise_for_status()
logging.debug("Create requests session: Status Code: {0}".format(r.status_code))
logging.info("Successfully authenticated to {0}".format(self.ticketing_tool))
# We log an error if authentication was not successful, because rest of the HTTP requests will not succeed.
# Instead of using e.message, use e.args[0] instead to prevent DeprecationWarning for exception.message.
except requests.RequestException as e:
logging.error("Error authenticating to {0}. No valid kerberos principal found.".format(self.auth_url))
logging.error(e.args[0])
return s
def parse(self, inventory, loader, path, cache=True):
super(InventoryModule, self).parse(inventory, loader, path)
self.config_data = self._read_config_data(path)
auth_type = self.get_option("authentication")
if auth_type in ("basic", "ntlm"):
username = self.get_option("username")
password = self.get_option("password")
if not (username and password):
raise AnsibleError(("Username and password are required for "
"{} authentication.").format(auth_type))
if auth_type == "basic":
self._auth = (username, password)
elif auth_type == "ntlm":
if HAS_REQNTLM_AUTH:
self._auth = HTTPNTLMAuth(username, password)
else:
raise AnsibleError(
"The library 'requests_ntlm' is required for "
"NTLM authentication.")
elif auth_type == "kerberos":
if HAS_REQKRB_AUTH:
self._auth = HTTPKerberosAuth()
else:
raise AnsibleError(
"The library 'requests_kerberos' is required for "
"Kerberos authentication.")
self._build_inventory()
def extract_policy_cache(config, policy_cache_file=None, table_tag_file=None, column_tag_file=None, hdfs=False, ignore_list=[]):
"""
Functionality to sync Rangers view of tags with Atlas. Useful when Atlas database or the Kafka topic
ranger_entities_consumer has been dropped.
Reads a policycahe files with tags copied from Hives policy cache. Either output table and column tag files if those
parameters provided or else talk directly to Atlas and sync so Atlas is equal to the policy cache.
:param config: Config for the environment to talk to.
:param policy_cache_file: Input file, a policy cache file with tags from Hive.
:param table_tag_file: Output file for tags on tables or None if not desired.
:param column_tag_file: Output file for tags on columns or None if not desired.
:param hdfs: Set to true if also sync to HDFS paths. Not used if tag-files are set.
:param ignore_list: List of 'schema.table' to ignore.
:return: None
"""
policy_cache_data = json.load(open(policy_cache_file))
policy_cache = PolicyCache(policy_cache_data)
tables_dict = _remove_ignores(policy_cache.get_tags_for_all_tables(), ignore_list)
columns_dict = _remove_ignores(policy_cache.get_tags_for_all_columns(), ignore_list)
if table_tag_file is None and column_tag_file is None:
atlas_client = atlas.Client(config['atlas_api_url'], auth=HTTPKerberosAuth())
hive_client = None
if hdfs:
hive_client = hive.Client(config['hive_server'], config['hive_port'])
sync_client = tagsync.Sync(atlas_client, hive_client=hive_client)
sync_client.sync_table_tags(tables_dict, clear_not_listed=True)
sync_client.sync_column_tags(columns_dict, clear_not_listed=True)
if hdfs:
sync_client.sync_table_storage_tags(tables_dict, clear_not_listed=True)
elif table_tag_file and column_tag_file:
_write_table_tag_file(table_tag_file, tables_dict)
_write_column_tag_file(column_tag_file, columns_dict)
else:
raise AttributeError("Either both table tag and column tag files must be set or neither.")
def __init__(self,
client_id,
capture_request_history=False,
saml_auth=HTTPKerberosAuth(mutual_authentication=OPTIONAL),
expiration=120,
verify=True):
self.client_id = client_id
self.capture_request_history = capture_request_history
self.history = []
self.expiration = expiration # Defaults to 2 hours (120 min)
self.verify = verify
self.redirect_uri = 'urn:ietf:wg:oauth:2.0:oob'
# DOI SAML service required these headers for single-sign-on. Developers can explicity over-write if needed...
self.saml_headers = {
"User-Agent":
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET4.0C; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0E; InfoPath.3)"
}
# DOI SAML service did not support 'REQUIRED' mutual authentication. HTTPKerberosAuth threw a 'mutual authentication exception'. Developers can explicity over-write if needed (or even add NTLM or some other 3rd party auth handler to the SAML communications)
self.saml_auth = saml_auth
### Derived Fields ###
self._verify_cert = True
self._base_url = None # Expected to be - https://host/<instance>/sharing/rest where <instance> is optional
self._oauth_info = None # Derived from the portal "authorize" endpoint
self._saml_code = None # Derived from the SAML login
self._token_data = None # Derived from the portal "token" endpoint
self._token_acquired = None
def _create_requests_session(self):
"""
Creates a Requests Session and authenticates to base API URL with kerberos-requests.
We're using a Session to persist cookies across all requests made from the Session instance.
:return s: Requests Session.
"""
# TODO: Support other authentication methods.
# Set up authentication for requests session.
s = requests.Session()
if self.auth == 'kerberos':
self.principal = _get_kerberos_principal()
s.auth = HTTPKerberosAuth(mutual_authentication=DISABLED)
s.verify = self.verify
if isinstance(self.auth, tuple):
s.auth = self.auth
s.verify = self.verify
if hasattr(self, 'headers'):
s.headers = self.headers
# Proxy setup
if hasattr(self, 'proxies'):
s.proxies.update(self.proxies)
# Try to authenticate to auth_url.
try:
r = s.get(self.auth_url)
logger.debug("Create requests session: status code: {0}".format(
r.status_code))
r.raise_for_status()
logger.info("Successfully authenticated to {0}".format(
self.ticketing_tool))
return s
except requests.RequestException as e:
logger.error("Error authenticating to {0}".format(self.auth_url))
logger.error(e)
s.close()
def _initialize_resource_manager(self, **kwargs):
"""Initialize the Hadoop YARN Resource Manager instance used for this kernel's lifecycle."""
endpoints = None
if self.yarn_endpoint:
endpoints = [self.yarn_endpoint]
# Only check alternate if "primary" is set.
if self.alt_yarn_endpoint:
endpoints.append(self.alt_yarn_endpoint)
if self.yarn_endpoint_security_enabled:
from requests_kerberos import HTTPKerberosAuth
auth = HTTPKerberosAuth()
else:
# If we have the appropriate version of yarn-api-client, use its SimpleAuth class.
# This allows EG to continue to issue requests against the YARN api when anonymous
# access is not allowed. (Default is to allow anonymous access.)
try:
from yarn_api_client.auth import SimpleAuth
kernel_username = KernelSessionManager.get_kernel_username(
**kwargs)
auth = SimpleAuth(kernel_username)
self.log.debug(
f"Using SimpleAuth with '{kernel_username}' against endpoints: {endpoints}"
)
except ImportError:
auth = None
self.resource_mgr = ResourceManager(service_endpoints=endpoints,
auth=auth,
verify=cert_path)
self.rm_addr = self.resource_mgr.get_active_endpoint()
def _make_request(self, entity, params):
"""
Send a request to Pyxis
:param str entity: entity part to construct a full URL for request.
:param dict params: Pyxis query parameters.
:return: Json response from Pyxis
:rtype: dict
:raises PyxisRequestError: If Pyxis returns error as a response
"""
entity_url = urllib.parse.urljoin(self._api_root, entity)
auth_method = HTTPKerberosAuth(mutual_authentication=OPTIONAL)
response = requests.get(entity_url,
params=params,
auth=auth_method,
timeout=conf.net_timeout)
if response.ok:
return response.json()
# Warn early, in case there is an error in the error handling code below
log.warning("Request to %s gave %r", response.request.url, response)
try:
response_text = response.json()
except ValueError:
response_text = response.text
raise PyxisRequestError(response.status_code, response_text)
def _create_requests_session(self):
"""
Creates a Requests Session and authenticates to base API URL with HTTP Basic Auth or Kerberos Auth.
We're using a Session to persist cookies across all requests made from the Session instance.
:return s: Requests Session.
"""
s = requests.Session()
# Kerberos Auth
if self.auth == 'kerberos':
self.principal = ticket._get_kerberos_principal()
s.auth = HTTPKerberosAuth(mutual_authentication=DISABLED)
s.verify = False
# HTTP Basic Auth
if isinstance(self.auth, tuple):
username, password = self.auth
self.principal = username
s.params.update({'user': username, 'pass': password})
# Try to authenticate to auth_url.
try:
r = s.get(self.auth_url)
logger.debug("Create requests session: status code: {0}".format(r.status_code))
r.raise_for_status()
# Special case for RT. A 200 status code is still returned if authentication failed. Have to check r.text.
if '200' not in r.text:
raise requests.RequestException
logger.info("Successfully authenticated to {0}".format(self.ticketing_tool))
return s
except requests.RequestException as e:
logger.error("Error authenticating to {0}".format(self.auth_url))
s.close()
def get_image_advisories_from_image_nvrs(grouped_images):
nvr_to_image_erratum = {}
krb_auth = HTTPKerberosAuth()
nvrs_to_check = sum([len(nvrs) for nvrs in grouped_images.values()])
nvrs_checks = 0
print("Querying Errata for advisories belonging to %d images:" % nvrs_to_check)
for nvrs in sorted(grouped_images.values(), key=len, reverse=True):
for nvr in nvrs:
nvrs_checks += 1
if nvr in nvr_to_image_erratum:
continue
r = requests.get(ERRATA_URL + "build/" + nvr, auth=krb_auth)
r.raise_for_status()
data = r.json()
if not data['all_errata']:
# Super weird. This means we have a container that wasn't shipped via an advisory.
logging.warn("Failed to find errata for %s at %s" % (nvr, r.request.url))
continue
errata_id = data["all_errata"][0]["id"]
errata_name = data["all_errata"][0]["name"]
nvr_to_image_erratum[nvr] = errata_name
msg = "[%i/%i]: %s" % (nvrs_checks, nvrs_to_check, errata_name)
sys.stdout.write(msg + chr(8) * len(msg))
sys.stdout.flush()
r = requests.get(ERRATA_URL + "erratum/%s/builds" % (errata_name), auth=krb_auth)
r.raise_for_status()
data = r.json()
for builds_dict in data.values():
for builds_list in builds_dict["builds"]:
for next_nvr in builds_list.keys():
nvr_to_image_erratum[next_nvr] = [errata_id, errata_name, data.keys()]
return nvr_to_image_erratum
def get_facts_puppetdb(self, apiurl, facts, hostgroup):
url = '%s/facts' % apiurl
if 'v3' in apiurl:
query_base = '["and",["or",%s],["in", "certname", ["extract", "certname", ["select-facts", ["and", ["=", "name", "hostgroup"], ["~", "value", "%s"]]]]]]'
else:
query_base = '["and",["or",%s],["in", "certname", ["extract", "certname", ["select_facts", ["and", ["=", "name", "hostgroup"], ["~", "value", "%s"]]]]]]'
query_facts = ','.join(['["=","name","%s"]' % fact for fact in facts])
query = query_base % (query_facts, hostgroup)
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json, version=2'
}
payload = {'query': query}
logging.info("Getting facts from '%s', query: '%s'" % (url, query))
r = requests.get(url,
params=payload,
headers=headers,
auth=HTTPKerberosAuth())
if r.status_code == requests.codes.ok:
logging.info("Request code: '%s'" % r.status_code)
return json.loads(r.text)
else:
logging.error("The request failed with code '%s'" % r.status_code)
return None
