def handleResponse(self, data):
import urlparse
from twisted.internet import reactor
req = self.request
resp = self.response
resp.addbody(data)
m = req.get_method()
if not resp.status:
# Interrupted, or something like that.
log.msg('handleResponse without a status?', system='http')
return
if hasattr(self, 'handleResponse%d' % resp.status):
getattr(self, 'handleResponse%d' % resp.status)()
elif (resp.status in (301, 302, 303, 307) and m in ("GET", "HEAD")
or resp.status in (301, 302, 303) and m == "POST"):
# Some sane defaults
newurl = self.response.getheader('Location')
newurl = urlparse.urljoin(req.get_full_url(), newurl)
log.msg("redirecting %s request to %s" % (m, newurl))
newreq = URLRequest(newurl, data=req.data, headers=req.headers)
resdef, self.resdef = self.resdef, None
protocol.ClientCreator(
reactor, HTTPClient, newreq, resdef,
self.timeout).connectTCP(*splithostport(newreq))
elif (resp.status - resp.status % 100) in (400, 500, 600):
resdef, self.resdef = self.resdef, None
resdef.errback(HTTPError(resp.status, resp))
else:
resdef, self.resdef = self.resdef, None
resdef.callback(resp)
def downloadCRL(url, outFile=None, progressnotifier=None, blocksize=1024):
"""
:return:
CertificateList ASN.1 object
"""
# Open HTTP connection
headers = {'Accept': 'application/pkix-crl', 'User-Agent': USER_AGENT}
req = URLRequest(url, headers=headers)
res = urlopen(req, timeout=1)
# Download in chunks
totalBytes = int(res.info().getheader('Content-Length').strip())
bytesRead = 0
data = str()
while bytesRead < totalBytes:
# Read chunk
bytesToRead = blocksize
if totalBytes - bytesRead < bytesToRead:
bytesToRead = totalBytes - bytesRead
data += res.read(bytesToRead)
bytesRead = len(data)
# Update progress
if progressnotifier: progressnotifier(bytesRead, totalBytes)
# Write to file if specified
if outFile:
with open(outFile, 'w') as f:
f.write(data)
return CertificateList.fromBitString(data)
def sendRequest( self, request, timeout = None, log = False, log_data = False):
if log:
print(">" * 32)
print("[%s]>> REQUEST: %s, method: %s, data:" % (datetime.datetime.now().time(), request.get_full_url(), request.get_method()) )
if log_data:
pprint.pprint( request.get_data() )
if timeout is None:
timeout = 3600
try:
response = self.url_opener.open( request, timeout = timeout )
except HTTPError as err:
if err.code == 404:
raise RestNotFoundError( "Wrong URL: %s" % (request.get_full_url(),) )
elif err.code == 401:
try:
self.connect()
req = URLRequest( request.get_full_url(), request.get_data(), request.headers )
req.get_method = request.get_method
self._addEmcHeaders( req )
response = self.url_opener.open( req, timeout = timeout )
except HTTPError as err:
raise RestJsonError( err.fp.read() )
else:
raise RestJsonError( err.fp.read() )
self._saveCsrfToken( response )
return response
def create( self, name, payloadPath = None, payloadContent = None ):
'''Create a new payload.
Args:
name (str): the name of the payload to create.
payloadPath (str): path to the file containing the payload.
payloadContent (bytes): content of the new payload.
'''
if payloadPath is None and payloadContent is None:
raise LcApiException( 'no payload content or path specified' )
data = self._manager._apiCall( 'payload/%s/%s' % ( self._manager._oid, name ), POST, {} )
putUrl = data.get( 'put_url', None )
if putUrl is None:
return None
if payloadContent is None:
with open( payloadPath, 'rb' ) as f:
payloadContent = f.read()
request = URLRequest( str( putUrl ), headers = {
'Content-Type' : 'application/octet-stream'
} )
request.get_method = lambda: "PUT"
u = urlopen( request, data = payloadContent )
return u.read()
def handleResponse401(self):
from twisted.internet import reactor
# Plug in a basic auth handler by default
req = self.request
resp = self.response
auth = resp.getheader('WWW-Authenticate')
if not auth:
log.err("no auth header in 401 response")
resdef, self.resdef = self.resdef, None
resdef.errback(ValueError('401 without auth!'))
return
scheme, challenge = auth.split(' ', 1)
authmeth = 'get%sAuthResponse' % (scheme.capitalize())
if hasattr(self, authmeth):
authresp = getattr(self, authmeth)(challenge)
if authresp:
resdef, self.resdef = self.resdef, None
newreq = URLRequest(req.get_full_url(),
data=req.data,
headers=req.headers)
newreq.add_header('Authorization',
'%s %s' % (scheme, authresp))
protocol.ClientCreator(
reactor, HTTPClient, newreq, resdef,
self.timeout).connectTCP(*splithostport(newreq))
else:
log.err("auth scheme %s not supported" % scheme)
resdef, self.resdef = self.resdef, None
resdef.errback(
ValueError('401 auth scheme %s not supported' % scheme))
return
def _restCall(self,
url,
verb,
params,
altRoot=None,
queryParams=None,
rawBody=None,
contentType=None,
isNoAuth=False):
try:
if not isNoAuth:
headers = {"Authorization": "bearer %s" % self._jwt}
else:
headers = {}
if altRoot is None:
url = '%s/%s/%s' % (ROOT_URL, API_VERSION, url)
else:
url = '%s/%s' % (altRoot, url)
if queryParams is not None:
url = '%s?%s' % (url, urlencode(queryParams))
request = URLRequest(url,
rawBody if rawBody is not None else urlencode(
params, doseq=True).encode(),
headers=headers)
request.get_method = lambda: verb
request.add_header('User-Agent', 'lc-py-api')
if contentType is not None:
request.add_header('Content-Type', contentType)
u = urlopen(request)
try:
data = u.read()
if 0 != len(data):
resp = json.loads(data.decode())
else:
resp = {}
except ValueError as e:
LcApiException("Failed to decode data from API: %s" % e)
u.close()
ret = (200, resp)
except HTTPError as e:
errorBody = e.read()
try:
ret = (e.getcode(), json.loads(errorBody.decode()))
except:
ret = (e.getcode(), errorBody)
self._printDebug("%s: %s ( %s ) ==> %s ( %s )" %
(verb, url, str(params), ret[0], str(ret[1])))
return ret
def __login( self ):
url = self.request_url + '/index.html'
cj = cookielib.CookieJar()
url_opener = build_opener( HTTPCookieProcessor(cj) )
self.url_opener = url_opener
response = url_opener.open( URLRequest( url ) )
location = response.headers.get('Location', None)
if location:
request = URLRequest( location )
else: # already open
request = URLRequest( url )
request.add_header('X-EMC-REST-CLIENT', 'TRUE')
request.add_header('Authorization', b'Basic ' + base64.b64encode( self.user + b':' + self.password ))
request.add_header('WWW_Authenticate', 'Basic realm="Security Realm"')
# login and store cookuies
try:
url_opener.open( request )
except HTTPError as err:
if err.code == 401:
request.add_header('Authorization', b'Basic ' + base64.b64encode( self.user + b':' + "Password123#" ))
url_opener.open( request )
def urlopen(url, factory=None, timeout=300):
from twisted.internet import reactor
if factory is None:
factory = HTTPClientFactory
resdef = defer.Deferred()
if isinstance(url, basestring):
url = URLRequest(url)
host, port = splithostport(url)
f = factory(url, resdef, timeout)
resdef = reactor.connectTCP(host, port, f)
return f.deferred
def fetch_url(url):
""" Fetch URL using sane UA and encoding processing"""
request = URLRequest(
url, None, {
'User-Agent': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8)'
'AppleWebKit/536.25 (KHTML, like Gecko)'
'Version/6.0 Safari/536.25')
})
opener = build_opener(HTTPCookieProcessor(CookieJar()))
with closing(opener.open(request)):
data = opener.open(request).read()
enc = detect(data).get('encoding')
if enc:
data = data.decode(enc).encode('utf8')
return data
def ocspGetRequest(url, ocspReq, timeout=1):
if url[-1] == '/': url = url[:-1]
data = ocspReq
if isinstance(ocspReq, OCSPRequest):
data = ocspReq.toBitString()
headers = {
'Accept' : 'application/ocsp-response',
'User-Agent' : USER_AGENT
}
encoded = urlquote.quote_plus(b64encode(data))
req = URLRequest('{}/{}'.format(url, encoded), headers=headers)
res = urlopen(req, timeout=timeout)
if res:
return OCSPResponse.fromBitString(res.read())
return None
def ocspPostRequest(url, ocspReq, timeout=1):
if url[-1] == '/': url = url[:-1]
data = ocspReq
if isinstance(ocspReq, OCSPRequest):
data = ocspReq.toBitString()
headers = {
'Content-Type' : 'application/ocsp-request',
'Accept' : 'application/ocsp-response',
'User-Agent' : USER_AGENT
}
req = URLRequest(url, data=data, headers=headers)
res = urlopen(req, timeout=timeout)
if res:
return OCSPResponse.fromBitString(res.read())
return None
def makeJsonRequest( self, rest_request ):
url = self.request_url + rest_request.path
request = URLRequest( url )
request.add_header('Content-Type', 'application/json')
request.add_header('Accept', 'application/json')
self._addEmcHeaders( request )
if rest_request.content:
if isinstance( rest_request.content, dict ):
content = json.dumps( rest_request.content )
else:
content = rest_request.content
request.add_data( content )
if rest_request.http_method:
request.get_method = lambda http_method = str(rest_request.http_method): http_method
return request
def get( self, name ):
'''Get a specific payload content.
Args:
name (str): the name of the payload to get.
'''
data = self._manager._apiCall( 'payload/%s/%s' % ( self._manager._oid, name ), GET, {} )
getUrl = data.get( 'get_url', None )
if getUrl is None:
return None
request = URLRequest( getUrl )
request.get_method = lambda: "GET"
u = urlopen( request )
return u.read()
def _refreshJWT(self):
try:
if self._secret_api_key is None:
raise Exception('No API key set')
authData = {"secret": self._secret_api_key}
if self._uid is not None:
authData['uid'] = self._uid
else:
authData['oid'] = self._oid
request = URLRequest(API_TO_JWT_URL, urlencode(authData).encode())
request.get_method = lambda: "POST"
u = urlopen(request)
self._jwt = json.loads(u.read().decode())['jwt']
u.close()
except Exception as e:
self._jwt = None
raise LcApiException('Failed to get JWT from API key: %s' % e)
def makeUploadRequest( self, path, filename, data ):
if not path.startswith('/'):
path = '/' + path
request = URLRequest( self.request_url + path )
BOUNDARY = mimetools.choose_boundary()
CRLF = '\r\n'
content = []
content.append('--' + BOUNDARY)
content.append('Content-Disposition: form-data; name="upload"; filename="%s"' % (filename) )
content.append('Content-Type: application/octet-stream')
content.append('')
content.append(data)
content.append('--' + BOUNDARY + '--')
content = CRLF.join( content )
content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
request.add_unredirected_header('Content-Type', content_type)
request.add_unredirected_header('Content-Length', str(len(content)))
request.add_data( content )
return request
def _getXMLetree(self, xmlpath, noquote=0, timeout=10):
""" take a URL path, download XLM and return parsed Etree """
if noquote:
xurl = self.baseurl + xmlpath
else:
xurl = self.baseurl + URLQuote(xmlpath)
if self.debug & 0x02:
print("_getXMLetree: " + xurl)
# print("_getXMLetree: URLRequest")
req = URLRequest(xurl)
# print("_getXMLetree: self._opener.open ")
# HTTPError
try:
res = self._opener.open(req, None, timeout)
data = res.read()
# print("res.getcode() ", res.getcode(), len(data))
res.close()
except HTTPError as e:
self.error_str = str("Reponse Code: {0} : {1}").format(
e.code, xurl)
return None
if len(self.error_str):
self.error_str = ""
if self.debug & 0x200:
print(res.info())
print(data)
et = None
if len(data):
try:
et = ET.fromstring(data)
except ET.ParseError as e:
print("Etree ParseError ")
print("data = ", data)
print("e.message = ", e.message)
# raise
finally:
return et
else:
return None
def _sendfile(self, filename="", data="", load="n"):
if filename.startswith('/'):
xurl = self.baseurl + "/file/upload" + filename + "?load=" + load
else:
xurl = self.baseurl + "/file/upload/" + filename + "?load=" + load
if self.debug & 0x02:
print("{0} xurl: {1}".format(__name__, xurl))
req = URLRequest(xurl, data,
{'Content-Type': 'application/xml; charset="utf-8"'})
try:
res = self._opener.open(req, None)
responce = res.read()
# print("res.getcode() ", res.getcode(), len(responce))
res.close()
except HTTPError as e:
# print "e.read: ", e.read()
mess = "{!s}: {!s}: {!s}".format("/file/upload", filename, e.code)
raise IsySoapError(mess, httperr=e)
else:
return responce
def upload(self,
filePath,
source=None,
hint=None,
payloadId=None,
allowMultipart=False,
originalPath=None):
'''Upload a log.
Args:
filePath (str): path to the file to upload.
source (str): optional source identifier for where the log came from.
hint (str): optional data format hint for the log.
payloadId (str): optional unique payload identifier for the log, used to perform idempotent uploads.
allowMultipart (bool): unused, if True will perform multi-part upload for large logs.
'''
if self._uploadUrl is None:
# Get the ingest URL from the API.
self._uploadUrl = self._lc.getOrgURLs()['logs']
headers = {
'Authorization':
'Basic %s' % (base64.b64encode(
('%s:%s' %
(self._lc._oid, self._accessToken)).encode()).decode(), )
}
if source is not None:
headers['lc-source'] = source
if hint is not None:
headers['lc-hint'] = hint
if payloadId is not None:
headers['lc-payload-id'] = payloadId
if originalPath is not None:
headers['lc-path'] = base64.b64encode(
os.path.abspath(originalPath).encode()).decode()
with open(filePath, 'rb') as f:
# Get the file size.
f.seek(0, 2)
fileSize = f.tell()
f.seek(0)
if MAX_UPLOAD_PART_SIZE > fileSize:
# Simple single-chunk upload.
request = URLRequest(str('https://%s/ingest' %
(self._uploadUrl, )),
data=f.read(),
headers=headers)
try:
u = urlopen(request)
except HTTPError as e:
raise Exception('%s: %s' % (str(e), e.read().decode()))
try:
response = json.loads(u.read().decode())
except:
response = {}
else:
# Multi-part upload.
partId = 0
if payloadId is None:
headers['lc-payload-id'] = str(uuid.uuid4())
while True:
chunk = f.read(MAX_UPLOAD_PART_SIZE)
if not chunk:
break
if len(chunk) != MAX_UPLOAD_PART_SIZE:
headers['lc-part'] = "done"
else:
headers['lc-part'] = str(partId)
request = URLRequest(str('https://%s/ingest' %
(self._uploadUrl, )),
data=chunk,
headers=headers)
try:
u = urlopen(request)
except HTTPError as e:
raise Exception('%s: %s' % (str(e), e.read().decode()))
try:
response = json.loads(u.read().decode())
except:
response = {}
partId += 1
return response
def soapcomm(self, cmd, **kwargs):
"""
takes a command name and a list of keyword arguments.
each keyword is converted into a xml element
"""
if not isinstance(cmd, str) or not len(cmd):
raise IsyValueError("SOAP Method name missing")
if self.debug & 0x02:
print("sendcomm: ", cmd)
soap_cmd = self._gensoap(cmd, **kwargs)
xurl = self.baseurl + "/services"
if self.debug & 0x02:
print("xurl = ", xurl)
print("soap_cmd = ", soap_cmd)
req = URLRequest(xurl, soap_cmd,
{'Content-Type': 'application/xml; charset="utf-8"'})
data = ""
try:
res = self._opener.open(req, None)
data = res.read()
if self.debug & 0x200:
print("res.getcode() ", res.getcode(), len(data))
print("data ", data)
res.close()
except HTTPError as e:
self.error_str = str("Reponse Code: {0} : {1} {2}").format(
e.code, xurl, cmd)
if ((cmd == "DiscoverNodes" and e.code == 803)
or (cmd == "CancelNodesDiscovery" and e.code == 501)
# or (cmd == "RemoveNode" and e.code == 501)
):
if self.debug & 0x02:
print("spacial case: {0} : {1}".format(cmd, e.code))
print("e.code = ", e.code)
print("e.msg = ", e.msg)
print("e.hdrs = ", e.hdrs)
print("e.filename = ", e.filename)
print("e.code = ", type(e.code), e.code)
print("\n")
return e.read()
if self.debug & 0x202:
print("e.code = ", type(e.code), e.code)
# print "e.read = ", e.read()
print("e = ", e)
print("data = ", data)
mess = "{!s}: {!s}: {!s}".format(cmd, kwargs, e.code)
# This a messy and should change
raise IsySoapError(mess, httperr=e)
else:
if len(self.error_str):
self.error_str = ""
if self.debug & 0x200:
print(data)
return data
def __init__(self, url, resdef, timeout=300):
if not isinstance(url, URLRequest):
url = URLRequest(url)
self.request = url
self.timeout = timeout
self.resdef = resdef
def makeDownloadRequest( self, path ):
if not path.startswith('/'):
path = '/' + path
return URLRequest( self.request_url + path )