def ssl_urlopen(url, data):
sslcon = None
try:
sslcon = ssl._create_unverified_context()
except Exception as e:
sslcon = None
if sslcon: return urlopen(url, data, context=sslcon).read()
return urlopen(CMSSDT_ES_QUERY, data).read()
def get_payload(index, query, scroll=0):
data = {'index':index, 'query':query, 'scroll':scroll}
sslcon = None
try:
sslcon = ssl._create_unverified_context()
except Exception as e:
sslcon = None
if sslcon: return urlopen(CMSSDT_ES_QUERY,json.dumps(data), context=sslcon).read()
else: return urlopen(CMSSDT_ES_QUERY,json.dumps(data)).read()
def get_payload(index, query, scroll=0):
data = {'index':index, 'query':query, 'scroll':scroll}
if getenv("USE_ES_CMSSDT7","false")=="true": data["es_server"]=ES7_SERVER
sslcon = None
try:
sslcon = ssl._create_unverified_context()
except Exception as e:
sslcon = None
if sslcon: return urlopen(CMSSDT_ES_QUERY,json.dumps(data).encode("ascii","ignore"), context=sslcon).read()
else: return urlopen(CMSSDT_ES_QUERY,json.dumps(data).encode("ascii","ignore")).read()
def get_payload(index, query, scroll=0):
data = {'index':index, 'query':query, 'scroll':scroll}
if scroll<=1: data['params'] = 'ignore_unavailable=true'
data["es_server"]=ES_SERVER
sslcon = None
try:
sslcon = ssl._create_unverified_context()
except Exception as e:
sslcon = None
if sslcon: return urlopen(CMSSDT_ES_QUERY,json.dumps(data).encode("ascii","ignore"), context=sslcon).read()
else: return urlopen(CMSSDT_ES_QUERY,json.dumps(data).encode("ascii","ignore")).read()
def get_payload(index, query, scroll=0):
data = {'index': index, 'query': query, 'scroll': scroll}
sslcon = None
try:
sslcon = ssl._create_unverified_context()
except Exception as e:
sslcon = None
if sslcon:
return urlopen(CMSSDT_ES_QUERY, json.dumps(data),
context=sslcon).read()
else:
return urlopen(CMSSDT_ES_QUERY, json.dumps(data)).read()
def github_api(uri, token, params=None, method="POST", headers=None, page=1, page_range=None):
if not params:
params = {}
if not headers:
headers = {}
if not page_range:
page_range = []
url = "https://api.github.com%s" % uri
data = ""
if method == "GET":
if params:
import urllib
url = url + "?" + urllib.urlencode(params)
else:
data = json.dumps(params)
if page > 1:
if not "?" in url:
url = url + "?"
else:
url = url + "&"
url = url + "page=" + str(page)
headers["Authorization"] = "token " + token
request = Request(url, data=data, headers=headers)
request.get_method = lambda: method
response = urlopen(request)
if page <= 1:
link = response.info().getheader("Link")
if link:
pages = [int(l.split("page=", 1)[1].split(">")[0]) for l in link.split(" ") if
"https://api.github.com" in l]
if len(pages) == 2:
page_range += range(pages[0], pages[1] + 1)
elif len(pages) == 1:
page_range += pages
return json.loads(response.read())
def github_api(uri, token, params=None, method="POST", headers=None, page=1, page_range=None):
if not params:
params = {}
if not headers:
headers = {}
if not page_range:
page_range = []
url = "https://api.github.com%s" % uri
data = ""
if method == "GET":
if params:
import urllib
url = url + "?" + urllib.urlencode(params)
else:
data = json.dumps(params)
if page > 1:
if not "?" in url:
url = url + "?"
else:
url = url + "&"
url = url + "page=" + str(page)
headers["Authorization"] = "token " + token
request = Request(url, data=data, headers=headers)
request.get_method = lambda: method
response = urlopen(request)
if (page <= 1) and (method=='GET'):
link = response.info().getheader("Link")
if link:
pages = [int(l.split("page=", 1)[1].split(">")[0]) for l in link.split(" ") if
"https://api.github.com" in l]
if len(pages) == 2:
page_range += range(pages[0], pages[1] + 1)
elif len(pages) == 1:
page_range += pages
return json.loads(response.read())
def send_request(uri,
payload=None,
passwd_file=None,
method=None,
es_ser=ES_SERVER,
ignore_doc=False):
header = {"Content-Type": "application/json"}
xuri = uri.split("/")
if (not ignore_doc) and (xuri[1] != "_doc"):
xuri[1] = "_doc"
uri = "/".join(xuri)
passwd = es_get_passwd(passwd_file)
if not passwd: return False
url = "%s/%s" % (es_ser, uri)
header['Authorization'] = 'Basic %s' % base64.b64encode(
"cmssdt:%s" % passwd)
try:
request = Request(url, payload, header)
if method: request.get_method = lambda: method
content = urlopen(request)
except Exception as e:
print("ERROR:", url, str(e))
print(payload)
return False
print("OK:", url)
return True
def send_request(uri,
payload=None,
passwd_file=None,
method=None,
es_ser=ES_SERVER,
ignore_doc=False):
header = {"Content-Type": "application/json"}
xuri = uri.split("/")
if (not ignore_doc) and (xuri[1] != "_doc"):
xuri[1] = "_doc"
uri = "/".join(xuri)
passwd = es_get_passwd(passwd_file)
if not passwd: return False
url = "%s/%s" % (es_ser, uri)
passman = HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, url, 'cmssdt', passwd)
auth_handler = HTTPBasicAuthHandler(passman)
opener = build_opener(auth_handler)
try:
install_opener(opener)
request = Request(url, payload, header)
if method: request.get_method = lambda: method
content = urlopen(request)
if method in ["POST", None]: print(content.read())
except Exception as e:
print("ERROR:", url, str(e))
print(payload)
return False
print("OK:", url)
return True
def update_crumb(jenkins_url, headers):
try:
req = Request(url=jenkins_url + '/crumbIssuer/api/json',
headers=headers)
crumb = loads(urlopen(req).read())
headers[crumb['crumbRequestField']] = crumb['crumb']
print("OK crumbRequest")
except HTTPError as e:
print("Running without Crumb Issuer: %s" % e)
pass
return headers
def process(opts):
xparam = []
for param in opts.params:
p,v=param.split("=",1)
xparam.append({"name":p,"value":v})
data = {"json":dumps({"parameter":xparam}),"Submit": "Build"}
try:
url = opts.server+'/job/'+opts.job+'/build'
data = urlencode(data)
req = Request(url=url,data=data,headers={"ADFS_LOGIN" : opts.user})
content = urlopen(req).read()
except Exception as e:
print("Unable to start jenkins job:",e)
def send_request(uri, payload=None, passwd_file=None, method=None):
passwd=es_get_passwd(passwd_file)
if not passwd: return False
url = "%s/%s" % (ES_SERVER,uri)
passman = HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None,url, 'cmssdt', passwd)
auth_handler = HTTPBasicAuthHandler(passman)
opener = build_opener(auth_handler)
try:
install_opener(opener)
request = Request(url, payload)
if method: request.get_method = lambda: method
content = urlopen(request)
except Exception as e:
print("ERROR:",url,str(e))
return False
return True
def send_request(uri, payload=None, passwd_file=None, method=None):
passwd = es_get_passwd(passwd_file)
if not passwd: return False
url = "%s/%s" % (ES_SERVER, uri)
passman = HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, url, 'cmssdt', passwd)
auth_handler = HTTPBasicAuthHandler(passman)
opener = build_opener(auth_handler)
try:
install_opener(opener)
request = Request(url, payload)
if method: request.get_method = lambda: method
content = urlopen(request)
except Exception as e:
print("ERROR:", url, str(e))
return False
return True
def build_jobs(jenkins_url, jobs_data, headers={}, user="******"):
for rk in ["OIDC_CLAIM_CERN_UPN"]:
if rk not in headers:
headers[rk] = user
install_opener(build_opener(HTTPCookieProcessor(CookieJar())))
for prams, job in jobs_data:
if not job: continue
headers = update_crumb(jenkins_url, headers)
url = jenkins_url + '/job/' + job + '/build'
data = {"json": prams, "Submit": "Build"}
try:
data = urlencode(data)
req = Request(url=url, data=data, headers=headers)
content = urlopen(req).read()
print("ALL_OK")
except Exception as e:
print("Unable to start jenkins job: %s" % e)
def send_request(uri, payload=None, passwd_file=None, method=None, es7=False):
es_ser = ES7_SERVER
header = {"Content-Type": "application/json"}
if not es7:
send_request(uri, payload, passwd_file, method, True)
header = {}
es_ser = ES_SERVER
passwd=es_get_passwd(passwd_file)
if not passwd: return False
url = "%s/%s" % (es_ser,uri)
passman = HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None,url, 'cmssdt', passwd)
auth_handler = HTTPBasicAuthHandler(passman)
opener = build_opener(auth_handler)
try:
install_opener(opener)
request = Request(url, payload, header)
if method: request.get_method = lambda: method
content = urlopen(request)
except Exception as e:
print("ERROR:",url,str(e))
return False
if es7: print("OK:",url)
return True
#!/usr/bin/env python
from __future__ import print_function
import sys, json
from os.path import expanduser
from _py2with3compatibility import Request, urlopen
from cms_static import GH_CMSSW_ORGANIZATION, GH_CMSSW_REPO
GH_TOKEN = open( expanduser("~/.github-token")).read().strip()
release_name = sys.argv[1]
branch = sys.argv[2]
print('Creating release:\n %s based on %s' % (release_name, branch))
# creating releases will be available in the next version of pyGithub
params = { "tag_name" : release_name,
"target_commitish" : branch,
"name" : release_name,
"body" : 'cms-bot is going to build this release',
"draft" : False,
"prerelease" : False }
request = Request("https://api.github.com/repos/" + GH_CMSSW_ORGANIZATION + "/" + GH_CMSSW_REPO +"/releases",
headers={"Authorization" : "token " + GH_TOKEN })
request.get_method = lambda: 'POST'
print('--')
try:
print(urlopen( request, json.dumps( params ) ).read())
print("OK release",release_name,"created")
except Exception as e:
print('There was an error while creating the release:\n', e)
import sys, json
from os.path import expanduser
from _py2with3compatibility import Request, urlopen
from cms_static import GH_CMSSW_ORGANIZATION, GH_CMSSW_REPO
GH_TOKEN = open(expanduser("~/.github-token")).read().strip()
release_name = sys.argv[1]
branch = sys.argv[2]
print('Creating release:\n %s based on %s' % (release_name, branch))
# creating releases will be available in the next version of pyGithub
params = {
"tag_name": release_name,
"target_commitish": branch,
"name": release_name,
"body": 'cms-bot is going to build this release',
"draft": False,
"prerelease": False
}
request = Request("https://api.github.com/repos/" + GH_CMSSW_ORGANIZATION +
"/" + GH_CMSSW_REPO + "/releases",
headers={"Authorization": "token " + GH_TOKEN})
request.get_method = lambda: 'POST'
print('--')
try:
print(urlopen(request, json.dumps(params)).read())
print("OK release", release_name, "created")
except Exception as e:
print('There was an error while creating the release:\n', e)
def get_template(index=''):
data = {'index':index, 'api': '/_template', 'prefix': True}
return urlopen(CMSSDT_ES_QUERY,json.dumps(data)).read()
def get_indexes(index='cmssdt-*'):
data = {'index':index, 'api': '/_cat', 'prefix': True}
return urlopen(CMSSDT_ES_QUERY,json.dumps(data)).read()
def github_api(uri,
token,
params=None,
method="POST",
headers=None,
page=1,
page_range=None,
raw=False,
per_page=None,
last_page=False):
if not params:
params = {}
if not headers:
headers = {}
if not page_range:
page_range = []
url = "https://api.github.com%s" % uri
data = ""
if per_page: params['per_page'] = per_page
if method == "GET":
if params:
url = url + "?" + urlencode(params)
else:
data = json.dumps(params)
if version_info[0] == 3:
data = data.encode("utf-8")
if page > 1:
if not "?" in url:
url = url + "?"
else:
url = url + "&"
url = url + "page=%s" % page
headers["Authorization"] = "token " + token
request = Request(url, data=data, headers=headers)
request.get_method = lambda: method
response = urlopen(request)
if (page <= 1) and (method == 'GET'):
link = ""
if version_info[0] == 2:
link = response.info().getheader("Link")
else:
link = response.info().get("Link")
if link:
pages = []
for x in link.split(" "):
m = re.match('^.*[?&]page=([1-9][0-9]*).*$', x)
if m: pages.append(int(m.group(1)))
if len(pages) == 2:
page_range += range(pages[0], pages[1] + 1)
elif len(pages) == 1:
page_range += pages
cont = response.read()
if raw: return cont
data = json.loads(cont)
if page_range and page <= 1:
if last_page:
return github_api(uri,
token,
params,
method,
headers,
page_range[-1],
page_range=None,
raw=False,
per_page=per_page,
last_page=last_page)
for page in page_range:
data += github_api(uri,
token,
params,
method,
headers,
page,
page_range=None,
raw=raw,
per_page=per_page,
last_page=last_page)
return data
def get_indexes(index='cmssdt-*'):
data = {'index':index, 'api': '/_cat', 'prefix': True}
return urlopen(CMSSDT_ES_QUERY,json.dumps(data)).read()
def get_template(index=''):
data = {'index':index, 'api': '/_template', 'prefix': True}
return urlopen(CMSSDT_ES_QUERY,json.dumps(data)).read()
def github_api(uri,
params=None,
method="POST",
headers=None,
page=1,
raw=False,
per_page=100,
last_page=False,
all_pages=True,
max_pages=-1):
global GH_RATE_LIMIT, GH_PAGE_RANGE
if max_pages > 0 and page > max_pages:
return '[]' if raw else []
if not params:
params = {}
if not headers:
headers = {}
url = "https://api.github.com%s" % uri
data = ""
if per_page and ('per_page' not in params): params['per_page'] = per_page
if method == "GET":
if params:
url = url + "?" + urlencode(params)
else:
data = json.dumps(params)
if version_info[0] == 3:
data = data.encode("utf-8")
if page > 1:
if not "?" in url:
url = url + "?"
else:
url = url + "&"
url = url + "page=%s" % page
headers["Authorization"] = "token " + get_gh_token()
request = Request(url, data=data, headers=headers)
request.get_method = lambda: method
response = urlopen(request)
if page <= 1: GH_PAGE_RANGE = []
try:
GH_RATE_LIMIT = [
int(response.headers["X-RateLimit-Remaining"]),
int(response.headers["X-RateLimit-Limit"]),
int(response.headers["X-Ratelimit-Reset"])
]
except Exception as e:
print("ERROR:", e)
if (page <= 1) and (method == 'GET'):
link = response.headers.get("Link")
if link:
pages = []
for x in link.split(" "):
m = re.match('^.*[?&]page=([1-9][0-9]*).*$', x)
if m: pages.append(int(m.group(1)))
if len(pages) == 2:
GH_PAGE_RANGE += range(pages[0], pages[1] + 1)
elif len(pages) == 1:
GH_PAGE_RANGE += pages
cont = response.read()
if raw: return cont
data = json.loads(cont)
if GH_PAGE_RANGE and all_pages:
if last_page:
return github_api(uri,
params,
method,
headers,
GH_PAGE_RANGE[-1],
raw=False,
per_page=per_page,
all_pages=False)
for page in GH_PAGE_RANGE:
if max_pages > 0 and page > max_pages: break
data += github_api(uri,
params,
method,
headers,
page,
raw=raw,
per_page=per_page,
all_pages=False)
return data
