def retrieve(self,
fileId,
fileVersion=-1,
pars=[],
hdrs={},
targetFile=None,
processing=None,
processingPars=None):
"""
Request file `fileId` from the NG/AMS Server, store it locally
in `targetFile`, and return the result of the operation as an
ngamsStatus object. If `targetFile` is a directory, the name of the
retrieved file is appended to the directory name.
If `file_version` is given then that specific of the version will be
retrieved.
If `processing` and `processingPars` are given, they are passed down
as the processing plug-in name and parameters to be applied to the
retrieved data *on the server side*, respectively.
"""
pars = list(pars)
pars.append(("file_id", fileId))
if fileVersion != -1:
pars.append(("file_version", str(fileVersion)))
if processing:
pars.append(("processing", processing))
if processingPars:
pars.append(("processingPars", processingPars))
targetFile = targetFile or '.'
resp, host, port = self._get('RETRIEVE', pars, hdrs)
host_id = "%s:%d" % (host, port)
with contextlib.closing(resp):
if resp.status != NGAMS_HTTP_SUCCESS:
return ngamsStatus.to_status(resp, host_id, 'RETRIEVE')
# If the target path is a directory, take the filename
# of the incoming data as the filename
fname = targetFile
if os.path.isdir(fname):
cdisp = resp.getheader('Content-Disposition')
parts = ngamsLib.parseHttpHdr(cdisp)
if 'filename' not in parts:
msg = "Missing or invalid Content-Disposition header in HTTP response"
raise Exception(msg)
fname = os.path.join(fname,
os.path.basename(parts['filename']))
# Dump the data into the target file
readf = functools.partial(resp.read, 65536)
with open(fname, 'wb') as f:
for buf in iter(readf, ''):
f.write(buf)
return ngamsStatus.dummy_success_stat(host_id)
def _handleCmdRetrieve(srvObj, reqPropsObj, httpRef):
"""
Carry out the action of a RETRIEVE command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of
actions done during the request handling
(ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
# For data files, retrieval must be enabled otherwise the request is
# rejected.
if (not srvObj.getCfg().getAllowRetrieveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Retrieve"])
raise Exception(errMsg)
# Previously this command allowed to retrieve the current logging file,
# the configuration file and any internal file. We don't do this anymore
# Get query information.
if 'ng_log' in reqPropsObj or 'cfg' in reqPropsObj or 'internal' in reqPropsObj:
raise Exception(
"ng_log, cfg and internal parameters not supported anymore")
# At least file_id must be specified if not an internal file has been
# requested.
if 'file_id' not in reqPropsObj or not reqPropsObj['file_id']:
errMsg = genLog("NGAMS_ER_RETRIEVE_CMD")
raise Exception(errMsg)
fileId = reqPropsObj.getHttpPar("file_id")
logger.debug("Handling request for file with ID: %s", fileId)
fileVer = -1
if (reqPropsObj.hasHttpPar("file_version")):
fileVer = int(reqPropsObj.getHttpPar("file_version"))
diskId = ""
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
hostId = ""
if (reqPropsObj.hasHttpPar("host_id")):
hostId = reqPropsObj.getHttpPar("host_id")
domain = ""
if (reqPropsObj.hasHttpPar("domain")):
domain = reqPropsObj.getHttpPar("domain")
quickLocation = True
if (reqPropsObj.hasHttpPar("quick_location")):
quickLocation = int(reqPropsObj.getHttpPar("quick_location"))
# First try the quick retrieve attempt, just try to get the first
# (and best?) suitable file which is online and located on a node in the
# same domain as the contacted node.
ipAddress = None
if (quickLocation):
location, host, ipAddress, port, mountPoint, filename,\
fileVersion, mimeType =\
ngamsFileUtils.quickFileLocate(srvObj, reqPropsObj, fileId,
hostId, domain, diskId,
fileVer)
# If not located the quick way try the normal way.
if (not ipAddress):
# Locate the file best suiting the query and send it back if possible.
location, host, ipAddress, port, mountPoint, filename, fileId,\
fileVersion, mimeType =\
ngamsFileUtils.locateArchiveFile(srvObj, fileId, fileVer,
diskId, hostId, reqPropsObj)
# If still not located, try to contact associated NGAS sites to query
# if the file is available there.
# TODO:
if (not ipAddress):
pass
if (location == NGAMS_HOST_LOCAL):
# Get the file and send back the contents from this NGAS host.
srcFilename = os.path.normpath("{0}/{1}".format(mountPoint, filename))
# Perform the possible file staging
performStaging(srvObj, reqPropsObj, httpRef, srcFilename)
# Perform the possible processing requested.
procResult = performProcessing(srvObj, reqPropsObj, srcFilename,
mimeType)
elif location in (NGAMS_HOST_CLUSTER, NGAMS_HOST_REMOTE) and \
srvObj.getCfg().getProxyMode():
logger.debug("NG/AMS Server acting as proxy - requesting file with ID: %s " +\
"from NG/AMS Server on host/port: %s/%s",
fileId, host, str(port))
# Act as proxy - get the file from the NGAS host specified and
# send back the contents. The file is temporarily stored in the
# Processing Area.
procDir = ngamsHighLevelLib.genProcDirName(srvObj.getCfg())
checkCreatePath(procDir)
pars = []
for par in reqPropsObj.getHttpParNames():
pars.append([par, reqPropsObj.getHttpPar(par)])
authHdr = ngamsSrvUtils.genIntAuthHdr(srvObj)
timeout = float(
reqPropsObj['timeout']) if 'timeout' in reqPropsObj else 60
conn = ngamsHttpUtils.httpGet(ipAddress,
port,
NGAMS_RETRIEVE_CMD,
pars=pars,
timeout=timeout,
auth=authHdr)
hdrs = {h[0]: h[1] for h in conn.getheaders()}
dataSize = int(hdrs["content-length"])
tmpPars = ngamsLib.parseHttpHdr(hdrs["content-disposition"])
dataFilename = tmpPars["filename"]
data = ngamsHttpUtils.sizeaware(conn, dataSize)
httpRef.send_data(data, mimeType, fname=dataFilename)
return
else:
# No proxy mode: A redirection HTTP response is generated.
httpRef.redirect(ipAddress, port)
return
# Send back reply with the result(s) queried and possibly processed.
genReplyRetrieve(srvObj, reqPropsObj, httpRef, procResult)