def buildHttpClient(url,
mimeType,
contentDisp,
contentLength,
checksum = None):
"""
construct the http client which sends file data to the remote next url
Returns: httplib
"""
# Separate the URL from the command.
idx = (url[7:].find("/") + 7)
tmpUrl = url[7:idx]
cmd = url[(idx + 1):]
http = httplib.HTTP(tmpUrl)
logger.debug("Sending HTTP header ...")
logger.debug("HTTP Header: %s: %s", NGAMS_HTTP_POST, cmd)
http.putrequest(NGAMS_HTTP_POST, cmd)
logger.debug("HTTP Header: Content-Type: %s", mimeType)
http.putheader("Content-Type", mimeType)
logger.debug("HTTP Header: Content-Disposition: %s", contentDisp)
http.putheader("Content-Disposition", contentDisp)
logger.debug("HTTP Header: Content-Length: %s", str(contentLength))
http.putheader("Content-Length", str(contentLength))
logger.debug("HTTP Header: Host: %s", getHostName())
http.putheader("Host", getHostName())
if (checksum):
http.putheader(NGAMS_HTTP_HDR_CHECKSUM, checksum)
http.endheaders()
logger.debug("HTTP header sent")
# rtobar, 14/3/16: the default timeout here was 1 [hr]! I'm keeping it like
# that, but probably it's too much
http._conn.sock.settimeout(3600)
return http
def _httpPostUrl(url,
mimeType,
contDisp = "",
dataRef = "",
dataSource = "BUFFER",
dataTargFile = "",
blockSize = 65536,
suspTime = 0.0,
timeOut = None,
authHdrVal = "",
dataSize = -1,
session_uuid = ""):
"""
Post the the data referenced on the given URL. This function is adapted from
ngamsLib.httpPostUrl, which does not support block-level suspension and cancelling for file transfer
The data send back from the remote server + the HTTP header information
is return in a list with the following contents:
[<HTTP status code>, <HTTP status msg>, <HTTP headers (list)>, <data>]
url: URL to where data is posted (string).
mimeType: Mime-type of message (string).
contDisp: Content-Disposition of the data (string).
dataRef: Data to post or name of file containing data to send
(string).
dataSource: Source where to pick up the data (string/BUFFER|FILE|FD).
dataTargFile: If a filename is specified with this parameter, the
data received is stored into a file of that name (string).
blockSize: Block size (in bytes) used when sending the data (integer).
suspTime: Time in seconds to suspend between each block (double).
timeOut: Timeout in seconds to wait for replies from the server
(double).
authHdrVal: Authorization HTTP header value as it should be sent in
the query (string).
dataSize: Size of data to send if read from a socket (integer).
Returns: List with information from reply from contacted
NG/AMS Server (reply, msg, hdrs, data) (list).
"""
T = TRACE()
# Separate the URL from the command.
idx = (url[7:].find("/") + 7)
tmpUrl = url[7:idx]
cmd = url[(idx + 1):]
http = httplib.HTTP(tmpUrl)
logger.debug("HTTP Header: %s: %s", NGAMS_HTTP_POST, cmd)
http.putrequest(NGAMS_HTTP_POST, cmd)
logger.debug("HTTP Header: Content-Type: %s", mimeType)
http.putheader("Content-Type", mimeType)
if (contDisp != ""):
logger.debug("HTTP Header: Content-Disposition: %s", contDisp)
http.putheader("Content-Disposition", contDisp)
if (authHdrVal):
if (authHdrVal[-1] == "\n"): authHdrVal = authHdrVal[:-1]
logger.debug("HTTP Header: Authorization: %s", authHdrVal)
http.putheader("Authorization", authHdrVal)
if (dataSource == "FILE"):
dataSize = getFileSize(dataRef)
elif (dataSource == "BUFFER"):
dataSize = len(dataRef)
if (dataSize != -1):
logger.debug("HTTP Header: Content-Length: %s", str(dataSize))
http.putheader("Content-Length", str(dataSize))
logger.debug("HTTP Header: Host: %s", getHostName())
http.putheader("Host", getHostName())
http.endheaders()
logger.debug("HTTP header sent")
http._conn.sock.settimeout(timeOut)
# Send the data.
logger.debug("Sending data ...")
if (dataSource == "FILE"):
fdIn = open(dataRef)
block = "-"
blockAccu = 0
http._conn.sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 87380) # to fit Fornax
while (block != ""):
"""
if (threadRunDic.has_key(session_uuid) and threadRunDic[session_uuid] == 0):
info(3, "Received cancel/suspend request, discard remaining blocks")
break
"""
block = fdIn.read(blockSize)
blockAccu += len(block)
http._conn.sock.sendall(block)
if (suspTime > 0.0): time.sleep(suspTime)
fdIn.close()
elif (dataSource == "FD"):
fdIn = dataRef
dataRead = 0
while (dataRead < dataSize):
if ((dataSize - dataRead) < blockSize):
rdSize = (dataSize - dataRead)
else:
rdSize = blockSize
block = fdIn.read(rdSize)
http._conn.sock.sendall(block)
dataRead += len(block)
if (suspTime > 0.0): time.sleep(suspTime)
else:
# dataSource == "BUFFER"
http.send(dataRef)
logger.debug("Data sent")
if (threadRunDic.has_key(session_uuid) and threadRunDic[session_uuid] == 0):
logger.debug("Received cancel/suspend request, close HTTP connection and return None values")
if (http != None):
http.close()
del http
return [None, None, None, None]
# Receive + unpack reply.
logger.debug("Waiting for reply ...")
reply, msg, hdrs = http.getreply()
if (hdrs == None):
errMsg = "Illegal/no response to HTTP request encountered!"
raise Exception(errMsg)
if (hdrs.has_key("content-length")):
dataSize = int(hdrs["content-length"])
else:
dataSize = 0
if (dataTargFile == ""):
data = http.getfile().read(dataSize)
else:
fd = None
try:
data = dataTargFile
fd = open(dataTargFile, "w")
fd.write(http.getfile().read(dataSize))
fd.close()
except Exception as e:
if (fd != None): fd.close()
raise e
# Dump HTTP headers if Verbose Level >= 4.
logger.debug("HTTP Header: HTTP/1.0 " + str(reply) + " " + msg)
if logger.isEnabledFor(logging.DEBUG):
for hdr in hdrs.keys():
logger.debug("HTTP Header: %s: %s", hdr, hdrs[hdr])
if (http != None):
http.close()
del http
return [reply, msg, hdrs, data]
def get_pronom_signature(type_):
"""
Get PRONOM signature.
Return latest signature file version number as int when `type_` equals
"version" or return latest signature XML file as string when `type_` equals
"file". Upon error, write to `stderr` and returls `False`.
"""
try:
soapVersionContainer = """<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"><soap:Body><getSignatureFileVersionV1 xmlns="http://pronom.nationalarchives.gov.uk" /></soap:Body></soap:Envelope>"""
soapFileContainer = """<?xml version="1.0" encoding="utf-8"?><soap:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"><soap:Body><getSignatureFileV1 xmlns="http://pronom.nationalarchives.gov.uk" /></soap:Body></soap:Envelope>"""
soapVersionHeader = """\"http://pronom.nationalarchives.gov.uk:getSignatureFileVersionV1In\""""
soapFileHeader = """\"http://pronom.nationalarchives.gov.uk:getSignatureFileV1In\""""
if type_ == "version":
soapAction = soapVersionHeader
soapStr = soapVersionContainer
elif type_ == "file":
soapAction = soapFileHeader
soapStr = soapFileContainer
else:
sys.stderr.write("get_pronom_signature(): unknown type: " + type_)
return False
webservice = http_client.HTTP("www.nationalarchives.gov.uk")
webservice.putrequest("POST", "/pronom/service.asmx")
webservice.putheader("Host", "www.nationalarchives.gov.uk")
webservice.putheader("User-Agent",
"PRONOM UTILS v{0} (OPF)".format(__version__))
webservice.putheader("Content-type", "text/xml; charset=\"UTF-8\"")
webservice.putheader("Content-length", "%d" % len(soapStr))
webservice.putheader("SOAPAction", soapAction)
try:
webservice.endheaders()
except Exception as e:
sys.stderr.write(
"get_pronom_signature(): failed to contact PRONOM;\n%s\n" %
(e))
sys.exit()
webservice.send(soapStr)
statuscode, statusmessage, header = webservice.getreply()
if statuscode == 200:
xml = webservice.getfile()
if type_ == "version":
exp = re.compile("\<Version\>([0-9]{1,4})\<\/Version\>")
sigxml = exp.search(xml.read())
if len(sigxml.group(1)) > 0:
return int(sigxml.group(1))
else:
sys.stderr.write(
"get_pronom_signature(): could not parse VERSION from SOAP response: "
+ type_)
return False
if type_ == "file":
exp = re.compile("\<SignatureFile\>.*\<\/SignatureFile\>")
sigxml = exp.search(xml.read())
sigtxt = sigxml.group(0) if sigxml else ''
if len(sigtxt) > 0:
tmpfile = "./tmp_getPronomSignature.xml"
with open(tmpfile, 'wb') as file_:
file_.write(
"""<?xml version="1.0" encoding="UTF-8"?>""" +
"\n")
file_.write(sigtxt)
if not check_well_formedness(tmpfile):
os.unlink(tmpfile)
sys.stderr.write(
"get_pronom_signature(): signaturefile not well formed"
)
return False
else:
os.unlink(tmpfile)
return """<?xml version="1.0" encoding="UTF-8"?>""" + "\n" + sigtxt
else:
sys.stderr.write(
"get_pronom_signature(): could not parse XML from SOAP response: "
+ type_)
return False
else:
sys.stderr.write("get_pronom_signature(): webservice error: '" +
str(statuscode) + " " + statusmessage + "'\n")
return False
sys.stderr.write("get_pronom_signature(): unexpected return")
return False
except Exception as e:
sys.stderr.write("get_pronom_signature(): unknown error: " + str(e))
return False
def open(self):
if self.scheme == 'http':
self.__http = http_client.HTTP(self.host, self.port)
else:
self.__http = http_client.HTTPS(self.host, self.port)