def _delFile(srvObj, path, hostId, execute): """ Delete (remove from disk) the file specified by the path on the given host. srvObj: Reference to NG/AMS server class object (ngamsServer). path: Path of file (string). hostId: ID of host where file is stored (string). Returns: Message with info about the execution (string). """ if (hostId and (hostId != srvObj.getHostId())): raise Exception, "DISCARD Command can only be executed locally!" # Handle the discard request locally. ngasHostId = "%s:%d" % (getHostName(), srvObj.getCfg().getPortNo()) if (os.path.exists(path)): if (not ngamsLib.fileRemovable(path)): return genLog("NGAMS_ER_DISCARD_NOT_REM", [path, ngasHostId]) if (not execute): msg = genLog("NGAMS_INFO_DISCARD_GRANTED", [path, ngasHostId]) else: rmFile(path) if (os.path.exists(path)): return genLog("NGAMS_ER_DISCARD_FAILED", [path, ngasHostId, "Not removable"]) else: msg = genLog("NGAMS_INFO_DISCARD_OK", [path, ngasHostId]) else: return genLog("NGAMS_ER_DISCARD_NOT_FOUND", [path, ngasHostId]) return msg
def ngamsRaiseEx_NGAMS_ER_DAPI_1(srvObj, reqPropsObj): """ Test DAPI, which raises an NGAMS_ER_DAPI Exception. Returns: Void. """ raise Exception, genLog("NGAMS_ER_DAPI", [genLog("NGAMS_ER_DB_COM", ["TEST EXCEPTION"])])
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle DISCARD Command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ if (reqPropsObj.hasHttpPar("help")): global _help return _help if (not srvObj.getCfg().getAllowRemoveReq()): errMsg = genLog("NGAMS_ER_ILL_REQ", ["Discard File"]) raise Exception(errMsg) diskId = None fileId = None fileVersion = None hostId = None path = None execute = 0 if (reqPropsObj.hasHttpPar("disk_id")): diskId = reqPropsObj.getHttpPar("disk_id") if (reqPropsObj.hasHttpPar("file_id")): fileId = reqPropsObj.getHttpPar("file_id") if (reqPropsObj.hasHttpPar("file_version")): try: fileVersion = int(reqPropsObj.getHttpPar("file_version")) except ValueError: raise Exception("Illegal value for File Version specified: " +\ str(fileVersion)) if (reqPropsObj.hasHttpPar("host_id")): hostId = reqPropsObj.getHttpPar("host_id") if (reqPropsObj.hasHttpPar("path")): path = reqPropsObj.getHttpPar("path") if (reqPropsObj.hasHttpPar("execute")): try: execute = int(reqPropsObj.getHttpPar("execute")) except: errMsg = genLog("NGAMS_ER_REQ_HANDLING", ["Must provide proper " +\ "value for parameter: execute (0|1)"]) raise Exception(errMsg) msg = _discardFile(srvObj, diskId, fileId, fileVersion, hostId, path, execute) if msg.startswith("NGAMS_INFO_"): status = NGAMS_SUCCESS else: status = NGAMS_FAILURE httpRef.send_status(msg, status=status, code=NGAMS_HTTP_SUCCESS)
def specificTreatment(fo): """ Method contains the specific treatment of the file passed from NG/AMS. fo: File object Returns: (file_id, finalFileName, type); The finalFileName is a string containing the name of the final file without extension. type is the mime-type from the header. """ import rfc822, cgi, re _EXT = '.msg' filename = fo.name uidTempl = re.compile( "^[uU][iI][dD]://[aAbBcCzZxX][0-9,a-z,A-Z]+(/[xX][0-9,a-z,A-Z]+){2}(#\w{1,}|/\w{0,}){0,}$" ) try: message = rfc822.Message(fo) type, tparams = cgi.parse_header(message["Content-Type"]) except Exception as e: err = "Parsing of mime message failed: " + str(e) errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE", [os.path.basename(filename), _PLUGIN_ID, err]) raise Exception(errMsg) try: almaUid = message["alma-uid"] except: try: almaUid = message["Content-Location"] except: err = "Mandatory alma-uid or Content-Location parameter not found in mime header!" errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE", [os.path.basename(filename), _PLUGIN_ID, err]) raise Exception(errMsg) if not uidTempl.match(almaUid): err = "Invalid alma-uid found in Content-Location: " + almaUid errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE", [os.path.basename(filename), _PLUGIN_ID, err]) raise Exception(errMsg) try: almaUid = almaUid.split('//', 2)[1].split('#')[0] if almaUid[-1] == '/': almaUid = almaUid[:-1] fileId = almaUid finalFileName = almaUid.replace('/', ':') except Exception as e: err = "Problem constructing final file name: " + str(e) errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE", [os.path.basename(filename), _PLUGIN_ID, err]) raise Exception(errMsg) return (fileId, finalFileName, type)
def _checkFileAccess(srvObj, reqPropsObj, httpRef, fileId, fileVersion=-1, diskId=""): """ Check if a file is accessible either on the local host or on a remotely located host. srvObj: Instance of the server object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). fileId: File ID (string). fileVersion: File Version (integer). diskId: Disk ID (string). Returns: Returns message indicating if the file is available (string). """ T = TRACE() logger.debug("Checking for access to file with ID: %s", fileId) # Check if the file is located on this host, or if the request should be # forwarded (if this server should act as proxy). location, fileHost, ipAddress, filePortNo, mountPoint, filename, fileId,\ fileVersion, mimeType =\ ngamsFileUtils.locateArchiveFile(srvObj, fileId, fileVersion, diskId) if (location != NGAMS_HOST_LOCAL): # Go and get it! host, port = srvObj.get_remote_server_endpoint(fileHost) pars = (('file_access', fileId), ('file_version', fileVersion), ('disk_id', diskId)) resp = ngamsHttpUtils.httpGet(host, port, 'STATUS', pars, timeout=60) with contextlib.closing(resp): return ngamsStatus.to_status(resp, fileHost, 'STATUS').getMessage() else: # First check if this system allows for Retrieve Requests. if (not srvObj.getCfg().getAllowRetrieveReq()): msg = genLog("NGAMS_INFO_SERVICE_UNAVAIL", ["File Retrieval"]) else: fileHost = "%s:%d" % (getHostName(), filePortNo) msg = genLog("NGAMS_INFO_FILE_AVAIL", [fileId + "/Version: " + str(fileVersion), fileHost]) return msg
def handlePars(reqPropsObj, parDic): """ Parse/handle the HTTP parameters. reqPropsObj: NG/AMS request properties object (ngamsReqProps). parDic: Dictionary with the parameters (dictionary). Returns: Void. """ T = TRACE() # Get parameters. logger.debug("Get request parameters") parDic[TARG_MIME_TYPE] = None parDic[FILE_ID] = None parDic[VERSIONING] = 1 parDic[COMPRESSION] = None parDic[COMPRESSION_EXT] = None if (reqPropsObj.hasHttpPar(TARG_MIME_TYPE)): parDic[TARG_MIME_TYPE] = reqPropsObj.getHttpPar(TARG_MIME_TYPE) # If the file_id is not given, we derive it from the name of the URI. if (reqPropsObj.hasHttpPar(FILE_ID)): parDic[FILE_ID] = reqPropsObj.getHttpPar(FILE_ID) if (not parDic[FILE_ID]): if (reqPropsObj.getFileUri().find("file_id=") > 0): file_id = reqPropsObj.getFileUri().split("file_id=")[1] parDic[FILE_ID] = file_id logger.info("No file_id given, but found one in the URI: %s", parDic[FILE_ID]) else: parDic[FILE_ID] = os.path.basename(reqPropsObj.getFileUri()) logger.info("No file_id given, using basename of URI: %s", parDic[FILE_ID]) if (reqPropsObj.hasHttpPar(VERSIONING)): parDic[VERSIONING] = int(reqPropsObj.getHttpPar(VERSIONING)) # Set also the no_versioning parameter for backwards compatibility reasons if (parDic[VERSIONING]): reqPropsObj.addHttpPar("no_versioning", "0") else: reqPropsObj.addHttpPar("no_versioning", "1") if (reqPropsObj.hasHttpPar(COMPRESSION)): parDic[COMPRESSION] = reqPropsObj.getHttpPar(COMPRESSION) if (reqPropsObj.hasHttpPar(COMPRESSION_EXT)): parDic[COMPRESSION_EXT] = reqPropsObj.getHttpPar(COMPRESSION_EXT) if ((parDic[COMPRESSION] and (parDic[COMPRESSION_EXT] == None)) or (not parDic[COMPRESSION] and (parDic[COMPRESSION_EXT] != None))): raise Exception, genLog("NGAMS_ER_DAPI", [ "Parameters compression and compression_ext" "must be given together." ])
def _handleCmdCRetrieve(srvObj, reqPropsObj, httpRef): """ Carry out the action of a CRETRIEVE command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ # For data files, retrieval must be enabled otherwise the request is # rejected. if (not srvObj.getCfg().getAllowRetrieveReq()): errMsg = genLog("NGAMS_ER_ILL_REQ", ["Retrieve"]) raise Exception(errMsg) # We don't allow processing yet if 'processing' in reqPropsObj: raise Exception('CRETRIEVE command does not allow processing (yet)') # At least container_id or container_name must be specified containerName = containerId = None if reqPropsObj.hasHttpPar("container_id") and reqPropsObj.getHttpPar( "container_id").strip(): containerId = reqPropsObj.getHttpPar("container_id").strip() if not containerId and reqPropsObj.hasHttpPar( "container_name") and reqPropsObj.getHttpPar( "container_name").strip(): containerName = reqPropsObj.getHttpPar("container_name").strip() if not containerId and not containerName: errMsg = genLog("NGAMS_ER_RETRIEVE_CMD") raise Exception(errMsg) # If container_name is specified, and maps to more than one container, # an error is issued if not containerId: containerId = srvObj.getDb().getContainerIdForUniqueName(containerName) logger.debug("Handling request for file with containerId: %s", containerId) # Build the container hierarchy, get all file references and send back the results container = srvObj.getDb().readHierarchy(containerId, True) cinfo = cinfo_from_database(container, srvObj, reqPropsObj) reader = ngamsMIMEMultipart.ContainerReader(cinfo) # Send all the data back httpRef.send_data(reader, NGAMS_CONT_MT)
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle REMDISK command. See also 'handleRemDisk()'. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ if (not srvObj.getCfg().getAllowRemoveReq()): errMsg = genLog("NGAMS_ER_ILL_REQ", ["Remove"]) raise Exception(errMsg) diskId = "" execute = 0 if (reqPropsObj.hasHttpPar("disk_id")): diskId = reqPropsObj.getHttpPar("disk_id") if (diskId == ""): errMsg = genLog("NGAMS_ER_CMD_SYNTAX", [NGAMS_REMDISK_CMD, "Missing parameter: disk_id"]) raise Exception(errMsg) if (not srvObj.getDb().diskInDb(diskId)): errMsg = genLog("NGAMS_ER_UNKNOWN_DISK", [diskId]) raise Exception(errMsg) if (reqPropsObj.hasHttpPar("execute")): try: execute = int(reqPropsObj.getHttpPar("execute")) except: errMsg = genLog("NGAMS_ER_REQ_HANDLING", ["Must provide proper " +\ "value for parameter: execute (0|1)"]) raise Exception(errMsg) # Carry out the command. status = remDisk(srvObj, reqPropsObj, diskId, execute) # Send reply back to requestor (if not already done). if (not status): return xmlStat = status.genXmlDoc(0, 1, 1, 1, 0) xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat) if (status.getStatus() == NGAMS_SUCCESS): httpStat = NGAMS_HTTP_SUCCESS else: httpStat = NGAMS_HTTP_BAD_REQ httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT, code=httpStat)
def extract_compression_params(reqPropsObj, plugin_pars): plugin_pars[COMPRESSION] = None plugin_pars[COMPRESSION_EXT] = None if COMPRESSION in reqPropsObj: plugin_pars[COMPRESSION] = reqPropsObj[COMPRESSION] if COMPRESSION_EXT in reqPropsObj: plugin_pars[COMPRESSION_EXT] = reqPropsObj[COMPRESSION_EXT] if ((plugin_pars[COMPRESSION] and plugin_pars[COMPRESSION_EXT] is None) or (not plugin_pars[COMPRESSION] and plugin_pars[COMPRESSION_EXT] is not None)): raise Exception, genLog("NGAMS_ER_DAPI", ["Parameters compression and compression_ext" "must be given together."])
def saveInStagingFile(ngamsCfgObj, reqPropsObj, httpRef, stagingFilename, diskInfoObj): """ Save the data ready on the HTTP channel, into the given Staging Area file. ngamsCfgObj: NG/AMS Configuration (ngamsConfig). reqPropsObj: NG/AMS Request Properties object (ngamsReqProps). stagingFilename: Staging Area Filename as generated by ngamsHighLevelLib.genStagingFilename() (string). diskInfoObj: Disk info object. Only needed if mutual exclusion is required for disk access (ngamsDiskInfo). Returns: Void. """ T = TRACE() try: blockSize = ngamsCfgObj.getBlockSize() return saveFromHttpToFile(ngamsCfgObj, reqPropsObj, httpRef, stagingFilename, blockSize, 1, diskInfoObj) except Exception as e: errMsg = genLog("NGAMS_ER_PROB_STAGING_AREA", [stagingFilename, str(e)]) logger.exception(errMsg) raise
def specificTreatment(fo): """ Method contains the specific treatment of the file passed from NG/AMS. fo: File object Returns: (file_id, finalFileName, type); The finalFileName is a string containing the name of the final file without extension. type is the mime-type from the header. """ import rfc822, cgi, re _EXT = '.msg' filename = fo.name # uidTempl = re.compile("[xX][0-9,a-f]{3,}/[xX][0-9,a-f]{1,}$") # This matches the old UID of type X0123456789abcdef/X01234567 uidTempl = re.compile( "^[uU][iI][dD]:/(/[xX][0-9,a-f,A-F]+){3}(#\w{1,}){0,}$") # This matches the new UID structure uid://X1/X2/X3#kdgflf try: message = rfc822.Message(fo) type, tparams = cgi.parse_header(message["Content-Type"]) except Exception, e: err = "Parsing of mime message failed: " + str(e) errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE", [os.path.basename(filename), _PLUGIN_ID, err]) raise Exception, errMsg
def checkFitsChecksum(reqPropsObj, stgFile=None): """ Carry out a check of the DATASUM and CHECKSUM for each HDU in the file. reqPropsObj: NG/AMS request properties object (ngamsReqProps). stgFile: If specified this is taken rather than from the Request Properties Object (ngamsReqProps). Returns: Void. """ if (not stgFile): stgFile = reqPropsObj.getStagingFilename() cmd = "chksumVerFitsChksum %s" % stgFile stat, out = ngamsPlugInApi.execCmd(cmd) if (out.find("Status: OK") == -1): if (out.find("Status: NOT OK") != -1): errMsg = genLog( "NGAMS_ER_DAPI_BAD_FILE", [stgFile, "checkFitsChecksum", "Illegal CHECKSUM/DATASUM"]) else: errMsg = "Problem found while checking file: %s. " %\ out.replace("\n", " ") # Backwards compatibility with old ESO FITS CHECKSUM scheme: If the # file pass the previous scheme, we consider it as OK and do not # return the failure. cmd = "chksumGenChecksum %s" % stgFile stat, out = ngamsPlugInApi.execCmd(cmd) if (out.strip().find("0/0000000000000000") == -1): raise Exception, errMsg chksumUtil = "chksumGenChecksum" else: chksumUtil = "chksumVerFitsChksum" logger.debug("File: %s checked with: %s. Result: OK", stgFile, chksumUtil)
def getFitsKeys(fitsFile, keyList): """ Get a FITS keyword from a FITS file. A dictionary is returned whereby the keys in the keyword list are the dictionary keys and the value the elements that these refer to. fitsFile: Filename of FITS file (string). keyList: Tuple of keys for which to extract values (tuple). Returns: Dictionary with the values extracted of the format: {<key 1>: [<val hdr 0>, <val hdr 1> ...], <key 2>: ...} (dictionary). """ T = TRACE() import astropy.io.fits as pyfits keyDic = defaultdict(list) try: for key in keyList: vals = pyfits.getval(fitsFile, key) if isinstance(vals, basestring): vals = [vals] keyDic[key] = list(vals) return keyDic except Exception, e: msg = ". Error: %s" % str(e) errMsg = genLog("NGAMS_ER_RETRIEVE_KEYS", [str(keyList), fitsFile + msg]) logger.exception(errMsg) raise
def getDpIdInfo(filename): """ Generate the File ID (here DP ID) for the file. filename: Name of FITS file (string). Returns: Tuple containing the value of ARCFILE, the DP ID of the file, and the JD date. The two latter deducted from the ARCFILE keyword (tuple). """ try: keyDic = getFitsKeys(filename, ["ARCFILE"]) arcFile = keyDic["ARCFILE"][0] els = arcFile.split(".") dpId = els[0] + "." + els[1] + "." + els[2] date = els[1].split("T")[0] # Make sure that the files are stored according to JD # (one night is 12am -> 12am). isoTime = '.'.join(els[1:]) ts1 = fromiso8601(isoTime) ts2 = tomjd(ts1) - 0.5 dateDirName = toiso8601(frommjd(ts2), fmt=FMT_DATE_ONLY) return [arcFile, dpId, dateDirName] except: err = "Did not find keyword ARCFILE in FITS file or ARCFILE illegal" errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE", [os.path.basename(filename), "ngamsFitsPlugIn", err]) logger.exception(errMsg) raise
def specificTreatment(fo): """ Method contains the specific treatment of the file passed from NG/AMS. fo: File object Returns: (file_id, finalFileName, type); The finalFileName is a string containing the name of the final file without extension. type is the mime-type from the header. """ import rfc822, cgi, re _EXT = '.msg' filename = fo.name uidTempl = re.compile( "^[uU][iI][dD]://[aAbBcCzZxX][0-9,a-z,A-Z]+(/[xX][0-9,a-z,A-Z]+){2}(#\w{1,}|/\w{0,}){0,}$" ) try: message = rfc822.Message(fo) type, tparams = cgi.parse_header(message["Content-Type"]) except Exception, e: err = "Parsing of mime message failed: " + str(e) errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE", [os.path.basename(filename), _PLUGIN_ID, err]) raise Exception, errMsg
def checkTarball(filename): """ Check that the tarball is correct filename: Name of tarball file (string). Returns: Void. """ err = 0 tmpFilename = filename.replace(":", "_") + "_tmp" try: execCmd("ln -s " + filename + " " + tmpFilename) stat, out, _ = execCmd("tar -tf " + tmpFilename) rmFile(tmpFilename) if (stat != 0): errMsg = str(out).replace("\n", " --- ") err = 1 except Exception as e: rmFile(tmpFilename) errMsg = str(e) err = 1 if (err): errMsg = "Error checking tarball: " + errMsg errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE", [filename, "ngasTarBallPlugIn", errMsg]) raise Exception(errMsg)
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle UNSUBSCRIBE command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ # added by [email protected] if (reqPropsObj.hasHttpPar("subscr_id")): id = reqPropsObj.getHttpPar("subscr_id") err, errStr = delSubscriber(srvObj, id) ########### else: if (reqPropsObj.hasHttpPar("url")): url = reqPropsObj.getHttpPar("url") else: errMsg = genLog("NGAMS_ER_CMD_SYNTAX", [NGAMS_SUBSCRIBE_CMD, "Missing parameter: url"]) raise Exception(errMsg) err, errStr = delSubscriber(srvObj, ngamsLib.getSubscriberId(url)) if err: httpRef.send_status('UNSUBSCRIBE command failed: ' + errStr, status=NGAMS_FAILURE, code=NGAMS_HTTP_SUCCESS) else: return "Successfully handled UNSUBSCRIBE command"
def ngamsRegisterGenericPlugIn(server_object, request_object, param_dict): """ Generic registration plug-in to handle registration of files :param server_object: Reference to NG/AMS Server Object (ngamsServer) :param request_object: NG/AMS request properties object (ngamsReqProps) :param param_dict: Parameter dictionary :return: Standard NG/AMS Data Archiving Plug-In Status as generated by ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus) """ logger.info("Register generic plug-in registering file with URI: %s", request_object.getFileUri()) disk_info = request_object.getTargDiskInfo() mime_type = request_object.getMimeType() stage_file = request_object.getStagingFilename() file_id = os.path.basename(request_object.getFileUri()) if mime_type == ngamsCore.NGAMS_UNKNOWN_MT: error_message = ngamsCore.genLog("NGAMS_ER_UNKNOWN_MIME_TYPE1", [file_id]) raise Exception(error_message) file_size = ngamsPlugInApi.getFileSize(stage_file) compression = "" uncompressed_size = file_size # Get various information about the file being handled file_version, relative_path, relative_filename, complete_filename, file_exists = \ ngamsPlugInApi.genFileInfoReg(server_object.getDb(), server_object.getCfg(), request_object, disk_info, stage_file, file_id) logger.info("Register generic plug-in finished processing file with URI %s: file_id=%s, file_version=%s, format=%s, file_size=%s", request_object.getFileUri(), file_id, file_version, mime_type, file_size) return ngamsPlugInApi.genRegPiSuccessStat(disk_info.getDiskId(), relative_filename, file_id, file_version, mime_type, file_size, uncompressed_size, compression, relative_path, disk_info.getSlotId(), file_exists, complete_filename)
def getDiskInfo(srvObj, reqPropsObj = None): """ Invokes the plug-in listed in the configuration to extract the information about the disks mounted. The information is returned in a dictionary which contains the Slot IDs as keys, and elements which each is an object of type ngamsPhysDiskInfo. srvObj: Reference to NG/AMS Server class instance (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Dictionary containing information about the disks (dictionary). """ plugIn = srvObj.getCfg().getOnlinePlugIn() logger.info("Invoking System Online Plug-In: %s(srvObj)", plugIn) plugInMethod = loadPlugInEntryPoint(plugIn) diskInfoDic = plugInMethod(srvObj, reqPropsObj) if not diskInfoDic: if (not ngamsLib.trueArchiveProxySrv(srvObj.getCfg())): errMsg = genLog("NGAMS_NOTICE_NO_DISKS_AVAIL", [srvObj.getHostId(), srvObj.getHostId()]) logger.warning(errMsg) logger.debug("Disk Dictionary: %s", str(diskInfoDic)) return diskInfoDic
def ngams_generic(ngams_server, request_properties): """ Data Archiving Plug-In to handle archiving of SDM multipart related message files containing ALMA UIDs in the Content-Location mime parameter or any other kind of file :param ngams_server: Reference to NG/AMS Server Object (ngamsServer) :param request_properties: NG/AMS request properties object (ngamsReqProps) :return: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus) """ logger.info("Mirroring plug-in handling data for file: %s", os.path.basename(request_properties.getFileUri())) # Create staging file disk_info = request_properties.getTargDiskInfo() staging_filename = request_properties.getStagingFilename() # request_properties format: /MIRRARCHIVE?mime_type=application/x-tar&filename=... file_format = request_properties.getMimeType() if not file_format: raise Exception("mime_type parameter not specified in MIRRARCHIVE request") # Example of file URI format: # http://ngas01.org:7777/RETRIEVE?disk_id=59622720f79296473f6106c15e5c2240&host_id=ngas01:7777&quick_location=1&file_version=1&file_id=backup.2011-02-02T22:01:59.tar file_id = request_properties.fileinfo["fileId"] file_version = request_properties.fileinfo["fileVersion"] # Specific treatment depending on the mime-type if file_format.find("multipart") >= 0 or file_format.find("multialma") >= 0: logger.debug("Mirroring plug-in applying specific treatment for multipart/multialma mime file") file_id, final_filename, file_format = specific_treatment(staging_filename) else: final_filename = file_id logger.debug("Mirroring plug-in processing request for file with URI %s, file_format=%s, file_id=%s, " "file_version=%s, final_filename=%s", request_properties.getFileUri(), file_format, file_id, file_version, final_filename) try: # Compression parameters uncompressed_size = ngamsPlugInApi.getFileSize(staging_filename) compression = "" today = ngamsCore.toiso8601(fmt=ngamsCore.FMT_DATE_ONLY) relative_path, relative_filename, complete_filename, file_exists = \ generate_file_info(ngams_server.getCfg(), disk_info, staging_filename, file_version, final_filename, [today]) # Make sure the format is defined if not file_format: file_format = ngamsPlugInApi.determineMimeType(ngams_server.getCfg(), staging_filename) file_size = ngamsPlugInApi.getFileSize(staging_filename) return ngamsPlugInApi.genDapiSuccessStat(disk_info.getDiskId(), relative_filename, file_id, file_version, file_format, file_size, uncompressed_size, compression, relative_path, disk_info.getSlotId(), file_exists, complete_filename) except Exception as e: raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE", [staging_filename, PLUGIN_ID, "Problem processing file in staging area: " + str(e)]))
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle REMFILE command. See also 'handleRemFile()'. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ T = TRACE() if (not srvObj.getCfg().getAllowRemoveReq()): errMsg = genLog("NGAMS_ER_ILL_REQ", ["Remove"]) raise Exception(errMsg) diskId = "" fileId = "" fileVersion = -1 execute = 0 if (reqPropsObj.hasHttpPar("disk_id")): diskId = reqPropsObj.getHttpPar("disk_id") if (reqPropsObj.hasHttpPar("file_id")): fileId = reqPropsObj.getHttpPar("file_id") if (reqPropsObj.hasHttpPar("file_version")): fileVersion = int(reqPropsObj.getHttpPar("file_version")) if (reqPropsObj.hasHttpPar("execute")): try: execute = int(reqPropsObj.getHttpPar("execute")) except: errMsg = genLog("NGAMS_ER_REQ_HANDLING", ["Must provide proper " +\ "value for parameter: execute (0|1)"]) raise Exception(errMsg) # Carry out the command. status = remFile(srvObj, reqPropsObj, diskId, fileId, fileVersion, execute) # Send reply back to requestor. xmlStat = status.genXmlDoc(0, 1, 1, 1, 0) xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat) httpRef.send_data(xmlStat, NGAMS_XML_MT)
def ngamsGenDapi(srvObj, reqPropsObj): """ Generic Data Archiving Plug-In to handle archiving of any file. srvObj: Reference to NG/AMS Server Object (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus). """ # For now the exception handling is pretty basic: # If something goes wrong during the handling it is tried to # move the temporary file to the Bad Files Area of the disk. logger.debug("Plug-In handling data for file: %s", os.path.basename(reqPropsObj.getFileUri())) try: parDic = {} handlePars(reqPropsObj, parDic) diskInfo = reqPropsObj.getTargDiskInfo() stgFile = reqPropsObj.getStagingFilename() ext = os.path.splitext(stgFile)[1][1:] # Generate file information. logger.debug("Generate file information") dateDir = toiso8601(fmt=FMT_DATE_ONLY) fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, reqPropsObj.\ getStagingFilename(), parDic[FILE_ID], parDic[FILE_ID], [dateDir]) complFilename, relFilename = checkForDblExt(complFilename, relFilename) # Compress the file if requested. uncomprSize, archFileSize, format, compression, comprExt =\ compressFile(srvObj, reqPropsObj, parDic) if (comprExt != ""): complFilename += ".%s" % comprExt relFilename += ".%s" % comprExt logger.debug( "DAPI finished processing file - returning to host application") return ngamsPlugInApi.genDapiSuccessStat( diskInfo.getDiskId(), relFilename, parDic[FILE_ID], fileVersion, format, archFileSize, uncomprSize, compression, relPath, diskInfo.getSlotId(), fileExists, complFilename) except Exception as e: msg = "Error occurred in DAPI: %s" % str(e) logger.error(msg) raise Exception(genLog("NGAMS_ER_DAPI_RM", [msg]))
def specific_treatment(file_path): """ Method contains the specific treatment of the file passed from NG/AMS :param file_path: File path :return: (file_id, final_filename, file_type); The final_filename is a string containing the name of the final file without extension. file_type is the mime-type from the header. """ filename = os.path.basename(file_path) mime_message = parse_multipart_primary_header(file_path) if mime_message is None: raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE", [filename, PLUGIN_ID, "Failed to parse MIME message"])) file_type = mime_message.get_content_type() alma_uid = mime_message["alma-uid"] if alma_uid is None: alma_uid = mime_message["Content-Location"] if alma_uid is None: raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE", [filename, PLUGIN_ID, "Mandatory 'alma-uid' and/or 'Content-Location' parameter not found in MIME header!"])) if UID_EXPRESSION.match(alma_uid) is None: raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE", [filename, PLUGIN_ID, "Invalid alma-uid found in Content-Location: " + alma_uid])) # Now, build final filename. We do that by looking for the UID in the message mime-header. # The final filename is built as follows: <ALMA-UID>.<EXT> where ALMA-UID has the slash character in the UID # replaced by colons. try: # Get rid of the 'uid://' and of anything following a '#' sign alma_uid = alma_uid.split("//", 2)[1].split("#")[0] alma_uid = alma_uid.replace("x", "X") file_id = alma_uid final_filename = alma_uid.replace("/", ":") except Exception as e: raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE", [filename, PLUGIN_ID, "Problem constructing final file name: " + str(e)])) return file_id, final_filename, file_type
def ngamsSdmMultipart(ngams_server, request_properties): """ Data Archiving Plug-In to handle archiving of SDM multipart related message files containing ALMA UIDs in the Content-Location mime parameter :param ngams_server: Reference to NG/AMS Server Object (ngamsServer) :param request_properties: NG/AMS request properties object (ngamsReqProps) :return: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus) """ # For now the exception handling is pretty basic: # If something goes wrong during the handling it is tried to move the temporary file to the bad-files directory logger.info("SDM multipart plug-in handling data for file: %s", os.path.basename(request_properties.getFileUri())) disk_info = request_properties.getTargDiskInfo() staging_filename = request_properties.getStagingFilename() file_id, final_filename, file_format = specific_treatment(staging_filename) if request_properties.hasHttpPar("file_id"): file_id = request_properties.getHttpPar("file_id") #if request_properties.hasHttpPar("file_version"): # file_version = request_properties.getHttpPar("file_version") logger.debug("SDM multipart plug-in processing request for file with URI %s, file_format=%s, file_id=%s, " "final_filename=%s", request_properties.getFileUri(), file_format, file_id, final_filename) try: # Compression parameters uncompressed_size = ngamsPlugInApi.getFileSize(staging_filename) compression = "" # Remember to update the temporary file name in the request properties object request_properties.setStagingFilename(staging_filename) today = ngamsCore.toiso8601(fmt=ngamsCore.FMT_DATE_ONLY) file_version, relative_path, relative_filename, complete_filename, file_exists = \ ngamsPlugInApi.genFileInfo(ngams_server.getDb(), ngams_server.getCfg(), request_properties, disk_info, staging_filename, file_id, final_filename, [today]) # Make sure the format is defined if not file_format: file_format = ngamsPlugInApi.determineMimeType(ngams_server.getCfg(), staging_filename) file_size = ngamsPlugInApi.getFileSize(staging_filename) return ngamsPlugInApi.genDapiSuccessStat(disk_info.getDiskId(), relative_filename, file_id, file_version, file_format, file_size, uncompressed_size, compression, relative_path, disk_info.getSlotId(), file_exists, complete_filename) except Exception as e: raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE", [staging_filename, PLUGIN_ID, "Problem processing file in staging area: " + str(e)]))
def __handleCmd(srvObj, reqPropsObj): """ Handle the Mirroring Archive (MIRRARCHIVE) Command. srvObj: Reference to NG/AMS server class object (ngamsServer).setState reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). Returns: Void. """ TRACE() # Is this NG/AMS permitted to handle Archive Requests? logger.debug( "Checking if this NG/AMS permitted to handle Archive Requests?") if (not srvObj.getCfg().getAllowArchiveReq()): errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"]) raise Exception, errMsg # Generate staging filename. stgFilename = reqPropsObj.getStagingFilename() logger.info("staging filename is: %s", stgFilename) startByte = 0 if (os.path.exists(stgFilename) == 0): logger.debug('this is a new staging file') else: startByte = os.path.getsize(stgFilename) logger.debug( 'staging file already exists, requesting resumption of download from byte %d', startByte) # Set reference in request handle object to the read socket. try: # Retrieve file_id and file_version from request proposal file_id = reqPropsObj.fileinfo['fileId'] file_version = reqPropsObj.fileinfo['fileVersion'] logger.debug("Got file_id=%s and file_version=%s", file_id, file_version) # Retrieve file contents (from URL, archive pull, or by storing the body # of the HTTP request, archive push). logger.info("Saving in staging file: %s", stgFilename) stagingInfo = saveInStagingFile(srvObj, srvObj.getCfg(), reqPropsObj, stgFilename, startByte) reqPropsObj.incIoTime(stagingInfo[0]) checksumPlugIn = "ngamsGenCrc32" checksum = stagingInfo[1] except ngamsFailedDownloadException.FailedDownloadException, e: raise
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handles the CAPPEND command @param srvObj: ngamsServer.ngamsServer @param reqPropsObj: ngamsLib.ngamsReqProps @param httpRef: ngamsLib.ngamsHttpRequestHandler """ # Check that we have been given either a containerId or a containerName containerId = containerName = None if reqPropsObj.hasHttpPar("container_id") and reqPropsObj.getHttpPar("container_id").strip(): containerId = reqPropsObj.getHttpPar("container_id").strip() elif reqPropsObj.hasHttpPar("container_name") and reqPropsObj.getHttpPar("container_name").strip(): containerName = reqPropsObj.getHttpPar("container_name").strip() if not containerId and not containerName: errMsg = genLog("NGAMS_ER_RETRIEVE_CMD") raise Exception(errMsg) # Check if we have been asked to force the operation force = False if reqPropsObj.hasHttpPar('force') and reqPropsObj.getHttpPar('force') == '1': force = True # Check if we have been asked to "close" the container closeContainer = False if reqPropsObj.hasHttpPar('close_container') and reqPropsObj.getHttpPar('close_container') == '1': closeContainer = True # If container_name is specified, and maps to more than one container, # (or to none) an error is issued containerIdKnownToExist = False if not containerId: containerId = srvObj.getDb().getContainerIdForUniqueName(containerName) containerIdKnownToExist = True # If necessary, check that the container exists if not containerIdKnownToExist: if not srvObj.getDb().containerExists(containerId): msg = "No container with containerId '" + containerId + "' found, cannot append files to it" raise Exception(msg) # If a single fileId has been given via URL parameters # and the request is a GET we update that single file # Otherwise, we assume a list of files is given in the # body of he request if reqPropsObj.getHttpMethod() == NGAMS_HTTP_GET: _handleSingleFile(srvObj, containerId, reqPropsObj, force, closeContainer) else: _handleFileList(srvObj, containerId, reqPropsObj, httpRef, force, closeContainer)
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle REARCHIVE command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ archive_start = time.time() # Execute the init procedure for the ARCHIVE Command. mimeType = ngamsArchiveUtils.archiveInitHandling(srvObj, reqPropsObj, httpRef) # If mime-type is None, the request has been handled, i.e., it might have # been a probe request or the server acting as proxy. if (not mimeType): return logger.debug("Archiving file: %s with mime-type: %s", reqPropsObj.getSafeFileUri(), mimeType) fileInfoObj, trgDiskInfoObj = receiveData(srvObj, reqPropsObj, httpRef) processRequest(srvObj, reqPropsObj, httpRef, fileInfoObj, trgDiskInfoObj) # If running as a cache archive, update the Cache New Files DBM # with the information about the new file. if (srvObj.getCachingActive()): fileVer = fileInfoObj.getFileVersion() ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, trgDiskInfoObj.getDiskId(), fileInfoObj.getFileId(), fileVer, fileInfoObj.getFilename()) # Create log/syslog entry for the successfulyl handled request. msg = genLog("NGAMS_INFO_FILE_ARCHIVED", [reqPropsObj.getSafeFileUri()]) msg = msg + ". Time: %.6fs" % (time.time() - archive_start) logger.info(msg, extra={'to_syslog': True}) srvObj.setSubState(NGAMS_IDLE_SUBSTATE) httpRef.send_ingest_status(msg, trgDiskInfoObj)
def notifyRegistrationService(srvObj, svrStatus='online'): """ to notify the ngas registration service that I am online now svrStatus = online|offline """ parDicOnline = ngamsPlugInApi.\ parseRawPlugInPars(srvObj.getCfg().getOnlinePlugInPars()) if (parDicOnline.has_key("regsvr_host")): if (svrStatus == "online"): errTag = "NGAMS_ER_ONLINE_PLUGIN" else: errTag = "NGAMS_ER_OFFLINE_PLUGIN" regsvr_host = parDicOnline["regsvr_host"] regsvr_port = parDicOnline["regsvr_port"] regsvr_path = parDicOnline["regsvr_path"] host_name = getHostName() host_port = srvObj.getCfg().getPortNo() body = urllib.urlencode({ 'ngas_host': host_name, 'ngas_port': host_port, 'status': svrStatus }) hdrs = {"Accept": "text/plain"} resp = ngamsHttpUtils.httpPost( regsvr_host, regsvr_port, regsvr_path, body, mimeType='application/x-www-form-urlencoded', hdrs=hdrs, timeout=10) with contextlib.closing(resp): if resp.status != 200: errMsg = "Problem notifying registration service! Error " + resp.reason errMsg = genLog(errTag, [errMsg]) logger.error(errMsg) #raise Exception, errMsg else: logger.debug("Successfully notified registration service: %s", svrStatus) logger.info(resp.read()) return
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle CLONE command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ T = TRACE() # Is this NG/AMS permitted to handle Archive Requests? if (not srvObj.getCfg().getAllowArchiveReq()): errMsg = genLog("NGAMS_ER_ILL_REQ", ["Clone"]) raise Exception(errMsg) # Check if State/Sub-State correct for perfoming the cloning. srvObj.checkSetState("Command CLONE", [NGAMS_ONLINE_STATE], [NGAMS_IDLE_SUBSTATE, NGAMS_BUSY_SUBSTATE]) # Get the parameters from the query. if (reqPropsObj.hasHttpPar("file_id")): fileId = reqPropsObj.getHttpPar("file_id") else: fileId ="" if (reqPropsObj.hasHttpPar("disk_id")): diskId = reqPropsObj.getHttpPar("disk_id") else: diskId = "" if (reqPropsObj.hasHttpPar("file_version")): fileVersion = int(reqPropsObj.getHttpPar("file_version")) else: fileVersion = -1 if (reqPropsObj.hasHttpPar("target_disk_id")): targetDiskId = reqPropsObj.getHttpPar("target_disk_id") else: targetDiskId = "" # Carry out the cloning. clone(srvObj, diskId, fileId, fileVersion,targetDiskId,reqPropsObj,httpRef)
def _handleCmdCRetrieve(srvObj, reqPropsObj, httpRef): """ Carry out the action of a CRETRIEVE command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ # For data files, retrieval must be enabled otherwise the request is # rejected. if (not srvObj.getCfg().getAllowRetrieveReq()): errMsg = genLog("NGAMS_ER_ILL_REQ", ["Retrieve"]) raise Exception(errMsg) # We don't allow processing yet if 'processing' in reqPropsObj: raise Exception('CRETRIEVE command does not allow processing (yet)') # Users can request a tarball instead of the default MIME multipart message return_tar = 'format' in reqPropsObj and reqPropsObj['format'] == 'application/x-tar' container_id = containers.get_container_id(reqPropsObj, srvObj.db) logger.debug("Handling request for file with containerId: %s", container_id) # Build the container hierarchy and get all file references container = srvObj.getDb().readHierarchy(container_id, True) cinfo = cinfo_from_database(container, srvObj, reqPropsObj) # Send all the data back, either as a multipart message or as a tarball if return_tar: httpRef.send_file_headers(cinfo.name, 'application/x-tar', tarsize_cinfo(cinfo)) send_toplevel_cinfo(cinfo, httpRef) else: reader = ngamsMIMEMultipart.ContainerReader(cinfo) httpRef.send_data(reader, NGAMS_CONT_MT)