def retrieve(self, fileId, fileVersion=-1, pars=[], hdrs={}, targetFile=None, processing=None, processingPars=None): """ Request file `fileId` from the NG/AMS Server, store it locally in `targetFile`, and return the result of the operation as an ngamsStatus object. If `targetFile` is a directory, the name of the retrieved file is appended to the directory name. If `file_version` is given then that specific of the version will be retrieved. If `processing` and `processingPars` are given, they are passed down as the processing plug-in name and parameters to be applied to the retrieved data *on the server side*, respectively. """ pars = list(pars) pars.append(("file_id", fileId)) if fileVersion != -1: pars.append(("file_version", str(fileVersion))) if processing: pars.append(("processing", processing)) if processingPars: pars.append(("processingPars", processingPars)) targetFile = targetFile or '.' resp, host, port = self._get('RETRIEVE', pars, hdrs) host_id = "%s:%d" % (host, port) with contextlib.closing(resp): if resp.status != NGAMS_HTTP_SUCCESS: return ngamsStatus.to_status(resp, host_id, 'RETRIEVE') # If the target path is a directory, take the filename # of the incoming data as the filename fname = targetFile if os.path.isdir(fname): cdisp = resp.getheader('Content-Disposition') parts = ngamsLib.parseHttpHdr(cdisp) if 'filename' not in parts: msg = "Missing or invalid Content-Disposition header in HTTP response" raise Exception(msg) fname = os.path.join(fname, os.path.basename(parts['filename'])) # Dump the data into the target file readf = functools.partial(resp.read, 65536) with open(fname, 'wb') as f: for buf in iter(readf, ''): f.write(buf) return ngamsStatus.dummy_success_stat(host_id)
def _handleCmdRetrieve(srvObj, reqPropsObj, httpRef): """ Carry out the action of a RETRIEVE command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ T = TRACE() # For data files, retrieval must be enabled otherwise the request is # rejected. if (not srvObj.getCfg().getAllowRetrieveReq()): errMsg = genLog("NGAMS_ER_ILL_REQ", ["Retrieve"]) raise Exception(errMsg) # Previously this command allowed to retrieve the current logging file, # the configuration file and any internal file. We don't do this anymore # Get query information. if 'ng_log' in reqPropsObj or 'cfg' in reqPropsObj or 'internal' in reqPropsObj: raise Exception( "ng_log, cfg and internal parameters not supported anymore") # At least file_id must be specified if not an internal file has been # requested. if 'file_id' not in reqPropsObj or not reqPropsObj['file_id']: errMsg = genLog("NGAMS_ER_RETRIEVE_CMD") raise Exception(errMsg) fileId = reqPropsObj.getHttpPar("file_id") logger.debug("Handling request for file with ID: %s", fileId) fileVer = -1 if (reqPropsObj.hasHttpPar("file_version")): fileVer = int(reqPropsObj.getHttpPar("file_version")) diskId = "" if (reqPropsObj.hasHttpPar("disk_id")): diskId = reqPropsObj.getHttpPar("disk_id") hostId = "" if (reqPropsObj.hasHttpPar("host_id")): hostId = reqPropsObj.getHttpPar("host_id") domain = "" if (reqPropsObj.hasHttpPar("domain")): domain = reqPropsObj.getHttpPar("domain") quickLocation = True if (reqPropsObj.hasHttpPar("quick_location")): quickLocation = int(reqPropsObj.getHttpPar("quick_location")) # First try the quick retrieve attempt, just try to get the first # (and best?) suitable file which is online and located on a node in the # same domain as the contacted node. ipAddress = None if (quickLocation): location, host, ipAddress, port, mountPoint, filename,\ fileVersion, mimeType =\ ngamsFileUtils.quickFileLocate(srvObj, reqPropsObj, fileId, hostId, domain, diskId, fileVer) # If not located the quick way try the normal way. if (not ipAddress): # Locate the file best suiting the query and send it back if possible. location, host, ipAddress, port, mountPoint, filename, fileId,\ fileVersion, mimeType =\ ngamsFileUtils.locateArchiveFile(srvObj, fileId, fileVer, diskId, hostId, reqPropsObj) # If still not located, try to contact associated NGAS sites to query # if the file is available there. # TODO: if (not ipAddress): pass if (location == NGAMS_HOST_LOCAL): # Get the file and send back the contents from this NGAS host. srcFilename = os.path.normpath("{0}/{1}".format(mountPoint, filename)) # Perform the possible file staging performStaging(srvObj, reqPropsObj, httpRef, srcFilename) # Perform the possible processing requested. procResult = performProcessing(srvObj, reqPropsObj, srcFilename, mimeType) elif location in (NGAMS_HOST_CLUSTER, NGAMS_HOST_REMOTE) and \ srvObj.getCfg().getProxyMode(): logger.debug("NG/AMS Server acting as proxy - requesting file with ID: %s " +\ "from NG/AMS Server on host/port: %s/%s", fileId, host, str(port)) # Act as proxy - get the file from the NGAS host specified and # send back the contents. The file is temporarily stored in the # Processing Area. procDir = ngamsHighLevelLib.genProcDirName(srvObj.getCfg()) checkCreatePath(procDir) pars = [] for par in reqPropsObj.getHttpParNames(): pars.append([par, reqPropsObj.getHttpPar(par)]) authHdr = ngamsSrvUtils.genIntAuthHdr(srvObj) timeout = float( reqPropsObj['timeout']) if 'timeout' in reqPropsObj else 60 conn = ngamsHttpUtils.httpGet(ipAddress, port, NGAMS_RETRIEVE_CMD, pars=pars, timeout=timeout, auth=authHdr) hdrs = {h[0]: h[1] for h in conn.getheaders()} dataSize = int(hdrs["content-length"]) tmpPars = ngamsLib.parseHttpHdr(hdrs["content-disposition"]) dataFilename = tmpPars["filename"] data = ngamsHttpUtils.sizeaware(conn, dataSize) httpRef.send_data(data, mimeType, fname=dataFilename) return else: # No proxy mode: A redirection HTTP response is generated. httpRef.redirect(ipAddress, port) return # Send back reply with the result(s) queried and possibly processed. genReplyRetrieve(srvObj, reqPropsObj, httpRef, procResult)