def votable_catalogue_plugin(srv, request): mime = request.getMimeType() parDic = ngamsPlugInApi.parseDapiPlugInPars(srv.cfg, mime) options = ('host', 'user', 'password', 'dbname') for option in options: if option not in parDic: raise Exception( '%s option not specified for votable_catalogue_plugin, please provide one' % option) host, user, password, dbname = [parDic[option] for option in options] dsn = 'host=%s user=%s password=%s dbname=%s' % (host, user, password, dbname) stageFilename = request.getStagingFilename() uncomprSize = ngamsPlugInApi.getFileSize(stageFilename) file_id = os.path.basename(request.getFileUri()) diskInfo = request.getTargDiskInfo() fileVersion, relPath, relFilename, complFilename, fileExists = \ ngamsPlugInApi.genFileInfo(srv.db, srv.cfg, request, diskInfo, request.getStagingFilename(), file_id, file_id, [], []) insert_catalogue_dataproduct(dsn, file_id) return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename, file_id, fileVersion, mime, uncomprSize, uncomprSize, '', relPath, diskInfo.getSlotId(), fileExists, complFilename)
def ngamsZHLDapi(srvObj, reqPropsObj): """ Generic Data Archiving Plug-In to handle archiving of any file. srvObj: Reference to NG/AMS Server Object (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus). """ T = TRACE() # For now the exception handling is pretty basic: # If something goes wrong during the handling it is tried to # move the temporary file to the Bad Files Area of the disk. logger.debug("Plug-In handling data for file: %s", os.path.basename(reqPropsObj.getFileUri())) try: parDic = {} handlePars(reqPropsObj, parDic) diskInfo = reqPropsObj.getTargDiskInfo() stgFile = reqPropsObj.getStagingFilename() ext = os.path.splitext(stgFile)[1][1:] # Generate file information. logger.debug("Generate file information") dateDir = toiso8601(fmt=FMT_DATE_ONLY) fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, reqPropsObj.\ getStagingFilename(), parDic[FILE_ID], parDic[FILE_ID], [dateDir]) complFilename, relFilename = checkForDblExt(complFilename, relFilename) # Compress the file if requested. uncomprSize, archFileSize, format, compression, comprExt =\ compressFile(srvObj, reqPropsObj, parDic) if (comprExt != ""): complFilename += ".%s" % comprExt relFilename += ".%s" % comprExt logger.debug( "DAPI finished processing file - returning to host application") insertFitsRecords(srvObj, reqPropsObj, stgFile) return ngamsPlugInApi.genDapiSuccessStat( diskInfo.getDiskId(), relFilename, parDic[FILE_ID], fileVersion, format, archFileSize, uncomprSize, compression, relPath, diskInfo.getSlotId(), fileExists, complFilename) except Exception: logger.exception("Error occurred in DAPI") raise
def ngamsSdmMultipart(ngams_server, request_properties): """ Data Archiving Plug-In to handle archiving of SDM multipart related message files containing ALMA UIDs in the Content-Location mime parameter :param ngams_server: Reference to NG/AMS Server Object (ngamsServer) :param request_properties: NG/AMS request properties object (ngamsReqProps) :return: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus) """ # For now the exception handling is pretty basic: # If something goes wrong during the handling it is tried to move the temporary file to the bad-files directory logger.info("SDM multipart plug-in handling data for file: %s", os.path.basename(request_properties.getFileUri())) disk_info = request_properties.getTargDiskInfo() staging_filename = request_properties.getStagingFilename() file_id, final_filename, file_format = specific_treatment(staging_filename) if request_properties.hasHttpPar("file_id"): file_id = request_properties.getHttpPar("file_id") #if request_properties.hasHttpPar("file_version"): # file_version = request_properties.getHttpPar("file_version") logger.debug("SDM multipart plug-in processing request for file with URI %s, file_format=%s, file_id=%s, " "final_filename=%s", request_properties.getFileUri(), file_format, file_id, final_filename) try: # Compression parameters uncompressed_size = ngamsPlugInApi.getFileSize(staging_filename) compression = "" # Remember to update the temporary file name in the request properties object request_properties.setStagingFilename(staging_filename) today = ngamsCore.toiso8601(fmt=ngamsCore.FMT_DATE_ONLY) file_version, relative_path, relative_filename, complete_filename, file_exists = \ ngamsPlugInApi.genFileInfo(ngams_server.getDb(), ngams_server.getCfg(), request_properties, disk_info, staging_filename, file_id, final_filename, [today]) # Make sure the format is defined if not file_format: file_format = ngamsPlugInApi.determineMimeType(ngams_server.getCfg(), staging_filename) file_size = ngamsPlugInApi.getFileSize(staging_filename) return ngamsPlugInApi.genDapiSuccessStat(disk_info.getDiskId(), relative_filename, file_id, file_version, file_format, file_size, uncompressed_size, compression, relative_path, disk_info.getSlotId(), file_exists, complete_filename) except Exception as e: raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE", [staging_filename, PLUGIN_ID, "Problem processing file in staging area: " + str(e)]))
def ngamsFitsPlugIn(srvObj, reqPropsObj): """ Data Archiving Plug-In to handle archiving of FITS files. srvObj: Reference to NG/AMS Server Object (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus). """ logger.info("Plug-In handling data for file with URI: %s", os.path.basename(reqPropsObj.getFileUri())) diskInfo = reqPropsObj.getTargDiskInfo() parDic = ngamsPlugInApi.parseDapiPlugInPars(srvObj.getCfg(), reqPropsObj.getMimeType()) # Check file (size + checksum) + extract information. dpId, dateDirName, comprExt = prepFile(reqPropsObj, parDic) # Get various information about the file being handled. dpIdInfo = getDpIdInfo(reqPropsObj.getStagingFilename()) dpId = dpIdInfo[1] dateDirName = dpIdInfo[2] fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, reqPropsObj.getStagingFilename(), dpId, dpId, [dateDirName], [comprExt]) # If a compression application is specified, apply this. uncomprSize, archFileSize, mime, compression, crc = compress( reqPropsObj, parDic) # Generate status + return. logger.debug( "DAPI finished processing of file - returning to main application") return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename, dpId, fileVersion, mime, archFileSize, uncomprSize, compression, relPath, diskInfo.getSlotId(), fileExists, complFilename, crc)
def ngasTarBallPlugIn(srvObj, reqPropsObj): """ Data Archiving Plug-In to handle archiving of tarballs. srvObj: Reference to NG/AMS Server Object (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus). """ stagingFilename = reqPropsObj.getStagingFilename() logger.info("Plug-In handling data for file with URI: %s", os.path.basename(reqPropsObj.getFileUri())) diskInfo = reqPropsObj.getTargDiskInfo() # Check file. checkTarball(stagingFilename) # Get various information about the file being handled. fileId = os.path.basename(reqPropsObj.getFileUri()) timestamp = fileId[fileId.find(".") + 1:] obsDay = tomjd(fromiso8601(timestamp)) - 0.5 dateDirName = toiso8601(obsDay, fmt=FMT_DATE_ONLY) baseFilename = fileId[0:-4] fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, stagingFilename, fileId, baseFilename, [dateDirName], []) # Generate status. logger.debug("Generating status ...") format = "application/x-tar" fileSize = ngamsPlugInApi.getFileSize(stagingFilename) logger.debug("DAPI finished processing of file") return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename, fileId, fileVersion, format, fileSize, fileSize, "NONE", relPath, diskInfo.getSlotId(), fileExists, complFilename)
def ngamsAlmaMultipart(srvObj, reqPropsObj): """ Data Archiving Plug-In to handle archiving of ALMA multipart related message files containing ALMA UIDs. srvObj: Reference to NG/AMS Server Object (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus). """ # For now the exception handling is pretty basic: # If something goes wrong during the handling it is tried to # move the temporary file to the Bad Files Area of the disk. logger.info("Plug-In handling data for file: " + os.path.basename(reqPropsObj.getFileUri())) diskInfo = reqPropsObj.getTargDiskInfo() stagingFilename = reqPropsObj.getStagingFilename() ext = os.path.splitext(stagingFilename)[1][1:] fo = open(stagingFilename, "r") (fileId, finalName, format) = specificTreatment(fo) fo.close() try: # Compress the file. uncomprSize = ngamsPlugInApi.getFileSize(stagingFilename) compression = "" # info(2,"Compressing file using: %s ..." % compression) # exitCode, stdOut = ngamsPlugInApi.execCmd("%s %s" %\ # (compression, # stagingFilename)) # if (exitCode != 0): # errMsg = _PLUGIN_ID+": Problems during archiving! " +\ # "Compressing the file failed" # raise Exception, errMsg # stagingFilename = stagingFilename + ".Z" # Remember to update the Temporary Filename in the Request # Properties Object. reqPropsObj.setStagingFilename(stagingFilename) # info(2,"File compressed") # ToDo: Handling of non-existing fileId # if (fileId == -1): # fileId = ngamsPlugInApi.genNgasId(srvObj.getCfg()) date = toiso8601(local=True, fmt=FMT_DATE_ONLY) fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, stagingFilename, fileId, finalName, [date]) # Generate status. logger.debug("Generating status ...") if not format: format = ngamsPlugInApi.determineMimeType(srvObj.getCfg(), stagingFilename) fileSize = ngamsPlugInApi.getFileSize(stagingFilename) return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename, fileId, fileVersion, format, fileSize, uncomprSize, compression, relPath, diskInfo.getSlotId(), fileExists, complFilename) except Exception as e: errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE", [os.path.basename(stagingFilename), _PLUGIN_ID, str(e)]) raise Exception(errMsg)
def ngamsMWAFitsPlugIn(srvObj, reqPropsObj): logger.info("ngamsMWAFitsPlugIn handling data for file with URI: %s", os.path.basename(reqPropsObj.getFileUri())) mime = reqPropsObj.getMimeType() parDic = ngamsPlugInApi.parseDapiPlugInPars(srvObj.getCfg(), mime) with DBPool.db_lock: if DBPool.db_pool is None: try: dbhost = parDic['dbhost'] dbname = parDic['dbname'] dbuser = parDic['dbuser'] dbpass = parDic['dbpassword'] except Exception as e: raise Exception( 'Must specify parameters dbhost, dbname, dbuser and dbpassword' ) maxconn = 8 try: maxconn = parDic['dbmaxconn'] except: pass logger.info("ngamsMWAFitsPlugIn creating database pool") DBPool.db_pool = psycopg2.pool.ThreadedConnectionPool( minconn=1, maxconn=maxconn, host=dbhost, user=dbuser, database=dbname, password=dbpass, port=5432) diskInfo = reqPropsObj.getTargDiskInfo() stageFilename = reqPropsObj.getStagingFilename() uncomprSize = ngamsPlugInApi.getFileSize(stageFilename) fileName = os.path.basename(reqPropsObj.getFileUri()) dpId = fileName obsid, obstime, box = split_file(fileName) fileVersion, relPath, relFilename, complFilename, fileExists = \ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, reqPropsObj.getStagingFilename(), dpId, dpId, [], []) logger.info("ngamsMWAFitsPlugIn version: %s path: %s relfilename: %s" % (fileVersion, relPath, relFilename)) sql = ( 'INSERT INTO data_files (observation_num, filetype, size, filename, site_path, host)' ' VALUES (%s, %s, %s, %s, %s, %s)') uri = 'http://mwangas/RETRIEVE?file_id=%s' % (fileName) logger.info('Inserting: %s', sql % (obsid, 8, uncomprSize, fileName, uri, box)) conn = None try: conn = DBPool.db_pool.getconn() conn.set_isolation_level( psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED) with conn.cursor() as cur: try: cur.execute(sql, [obsid, 8, uncomprSize, fileName, uri, box]) except Exception as e: ''' Do not want to raise an exception here because the sky data is more important than the metadata i.e. we want to keep the sky data even if there is a database problem. If there is an issue with an INSERT then go through and add it manually later (very unlikely to happen)''' conn.rollback() logger.error('Insert error: %s Message: %s', sql % (obsid, 8, uncomprSize, fileName, uri, box), str(e)) else: conn.commit() except Exception: logger.exception('Insert error: %s', sql % (obsid, 8, uncomprSize, fileName, uri, box)) finally: if conn: DBPool.db_pool.putconn(conn=conn) return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename, dpId, fileVersion, mime, uncomprSize, uncomprSize, '', relPath, diskInfo.getSlotId(), fileExists, complFilename)
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle the CARCHIVE Command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: (fileId, filePath) tuple. """ T = TRACE() # Is this NG/AMS permitted to handle Archive Requests? logger.debug("Is this NG/AMS permitted to handle Archive Requests?") if (not srvObj.getCfg().getAllowArchiveReq()): errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"]) raise Exception(errMsg) srvObj.checkSetState("Archive Request", [NGAMS_ONLINE_STATE], [NGAMS_IDLE_SUBSTATE, NGAMS_BUSY_SUBSTATE], NGAMS_ONLINE_STATE, NGAMS_BUSY_SUBSTATE, updateDb=False) if httpRef.command != 'POST': raise Exception("Only POST allowed for CARCHIVE") # Get mime-type (try to guess if not provided as an HTTP parameter). logger.debug( "Get mime-type (try to guess if not provided as an HTTP parameter).") if (reqPropsObj.getMimeType() == ""): mimeType = ngamsHighLevelLib.\ determineMimeType(srvObj.getCfg(), reqPropsObj.getFileUri()) reqPropsObj.setMimeType(mimeType) else: mimeType = reqPropsObj.getMimeType() # Determine the target volume, ignoring the stream concept. logger.debug("Determine the target volume, ignoring the stream concept.") targDiskInfo = ngamsArchiveUtils._random_target_volume(srvObj) if (targDiskInfo == None): errMsg = "No disk volumes are available for ingesting any files." raise Exception(errMsg) reqPropsObj.setTargDiskInfo(targDiskInfo) # Generate staging filename. logger.debug("Generate staging filename from URI: %s", reqPropsObj.getFileUri()) if (reqPropsObj.getFileUri().find("file_id=") >= 0): file_id = reqPropsObj.getFileUri().split("file_id=")[1] baseName = os.path.basename(file_id) else: baseName = os.path.basename(reqPropsObj.getFileUri()) stgFilename = os.path.join("/", targDiskInfo.getMountPoint(), NGAMS_STAGING_DIR, genUniqueId() + "___" + baseName) logger.debug("Staging filename is: %s", stgFilename) reqPropsObj.setStagingFilename(stgFilename) # Retrieve file contents (from URL, archive pull, or by storing the body # of the HTTP request, archive push). stagingInfo = saveInStagingFile(srvObj.getCfg(), reqPropsObj, httpRef, stgFilename, targDiskInfo) ioTime = stagingInfo[0] rootContainer = stagingInfo[1] fileDataList = stagingInfo[2] ingestRate = stagingInfo[3] reqPropsObj.incIoTime(ioTime) createContainers(rootContainer, None, srvObj) parDic = {} ngamsGenDapi.handlePars(reqPropsObj, parDic) diskInfo = reqPropsObj.getTargDiskInfo() # Generate file information. logger.debug("Generate file information") dateDir = toiso8601(fmt=FMT_DATE_ONLY) resDapiList = [] containerSizes = {} for item in fileDataList: container = item[0] filepath = item[1] crc = item[2] containerId = str(container.getContainerId()) basename = os.path.basename(filepath) fileId = basename fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, filepath, fileId, basename, [dateDir]) complFilename, relFilename = ngamsGenDapi.checkForDblExt( complFilename, relFilename) # Keep track of the total size of the container uncomprSize = ngamsPlugInApi.getFileSize(filepath) if containerId not in containerSizes: containerSizes[containerId] = 0 containerSizes[containerId] += uncomprSize mimeType = reqPropsObj.getMimeType() compression = "NONE" archFileSize = ngamsPlugInApi.getFileSize(filepath) resDapi = ngamsPlugInApi.genDapiSuccessStat( diskInfo.getDiskId(), relFilename, fileId, fileVersion, mimeType, archFileSize, uncomprSize, compression, relPath, diskInfo.getSlotId(), fileExists, complFilename) # Move file to final destination. logger.debug("Moving file to final destination") ioTime = mvFile(filepath, resDapi.getCompleteFilename()) reqPropsObj.incIoTime(ioTime) # Get crc info checksumPlugIn = "StreamCrc32" checksum = str(crc) # Get source file version # e.g.: http://ngas03.hq.eso.org:7778/RETRIEVE?file_version=1&file_id=X90/X962a4/X1 logger.debug("Get file version") file_version = resDapi.getFileVersion() if reqPropsObj.getFileUri().count("file_version"): file_version = int( (reqPropsObj.getFileUri().split("file_version=")[1] ).split("&")[0]) # Check/generate remaining file info + update in DB. logger.debug("Creating db entry") creDate = getFileCreationTime(resDapi.getCompleteFilename()) fileInfo = ngamsFileInfo.ngamsFileInfo().\ setDiskId(resDapi.getDiskId()).\ setFilename(resDapi.getRelFilename()).\ setFileId(resDapi.getFileId()).\ setFileVersion(file_version).\ setFormat(resDapi.getFormat()).\ setFileSize(resDapi.getFileSize()).\ setUncompressedFileSize(resDapi.getUncomprSize()).\ setCompression(resDapi.getCompression()).\ setIngestionDate(time.time()).\ setChecksum(checksum).setChecksumPlugIn(checksumPlugIn).\ setFileStatus(NGAMS_FILE_STATUS_OK).\ setCreationDate(creDate).\ setIoTime(reqPropsObj.getIoTime()) fileInfo.write(srvObj.getHostId(), srvObj.getDb()) # Add the file to the container srvObj.getDb().addFileToContainer(containerId, resDapi.getFileId(), True) # Update the container sizes for contSizeInfo in containerSizes.items(): srvObj.getDb().setContainerSize(contSizeInfo[0], contSizeInfo[1]) # Inform the caching service about the new file. logger.debug("Inform the caching service about the new file.") if (srvObj.getCachingActive()): diskId = resDapi.getDiskId() fileId = resDapi.getFileId() fileVersion = file_version filename = resDapi.getRelFilename() ngamsCacheControlThread.addEntryNewFilesDbm( srvObj, diskId, fileId, fileVersion, filename) # Update disk info in NGAS Disks. logger.debug("Update disk info in NGAS Disks.") srvObj.getDb().updateDiskInfo(resDapi.getFileSize(), resDapi.getDiskId()) resDapiList.append(resDapi) # Check if the disk is completed. # We use an approximate extimate for the remaning disk space to avoid # to read the DB. logger.debug("Check available space in disk") availSpace = getDiskSpaceAvail(targDiskInfo.getMountPoint(), smart=False) if (availSpace < srvObj.getCfg().getFreeSpaceDiskChangeMb()): targDiskInfo.setCompleted(1).setCompletionDate(time.time()) targDiskInfo.write(srvObj.getDb()) # Request after-math ... srvObj.setSubState(NGAMS_IDLE_SUBSTATE) msg = "Successfully handled Archive Pull Request for data file " +\ "with URI: " + reqPropsObj.getSafeFileUri() logger.info(msg) httpRef.send_ingest_status(msg, targDiskInfo) for resDapi in resDapiList: # Trigger Subscription Thread. This is a special version for MWA, in which we simply swapped MIRRARCHIVE and QARCHIVE # [email protected] logger.debug("triggering SubscriptionThread for file %s", resDapi.getFileId()) srvObj.addSubscriptionInfo( [(resDapi.getFileId(), resDapi.getFileVersion())], []) srvObj.triggerSubscriptionThread()
def ngamsNgLogPlugIn(srvObj, reqPropsObj): """ Data Archiving Plug-In to handle archiving of NG/AMS (OLAS style) log files. srvObj: Reference to NG/AMS Server Object (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus). """ # For now the exception handling is pretty basic: # If something goes wrong during the handling it is tried to # move the temporary file to the Bad Files Area of the disk. logger.debug("Plug-In handling data for file: %s", os.path.basename(reqPropsObj.getFileUri())) diskInfo = reqPropsObj.getTargDiskInfo() stagingFilename = reqPropsObj.getStagingFilename() ext = os.path.splitext(stagingFilename)[1][1:] # Now, build final filename. We do that by taking the date of # the first log entry. # # 2001-07-09T14:37:59.563 [INFO] Logging properties defined ... # # Alternatively the first entry could be of the form, e.g.: # # 2003-12-29T09:21:57.608 [INFO] LOG-ROTATE: 1072689717 - \ # SYSTEM-ID: ngamsArchiveClient@ngasdev2 # # In the former case the Log ID is equal to the NGAS ID. In the latter # case, the Log ID is equal to the System ID. # # The final filename is built as follows: <Log ID>.<date>.<ext> # # The file_id is: <Log ID>.<date> fo = open(stagingFilename, "r") firstLine = fo.readline() fo.close() try: # Compress the log file. uncomprSize = ngamsPlugInApi.getFileSize(stagingFilename) compression = "gzip" logger.debug("Compressing file using: %s ...", compression) exitCode, stdOut = ngamsPlugInApi.execCmd("%s %s" %\ (compression, stagingFilename)) if (exitCode != 0): errMsg = "ngamsNgLogPlugIn: Problems during archiving! " +\ "Compressing the file failed" raise Exception(errMsg) stagingFilename = stagingFilename + ".gz" # Remember to update the Temporary Filename in the Request # Properties Object. reqPropsObj.setStagingFilename(stagingFilename) logger.debug("Log file compressed") # Parse first line of the log file. timeStamp = firstLine.split(" ")[0] date = timeStamp.split("T")[0] sysIdIdx = firstLine.find("SYSTEM-ID") if (sysIdIdx != -1): logId = firstLine[sysIdIdx + len("SYSTEM-ID:"):].strip().\ split(" ")[0] else: logId = srvObj.getHostId() fileId = logId + "." + timeStamp fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, stagingFilename, fileId, fileId, [date]) # Generate status. logger.debug("Generating status ...") fformat = ngamsPlugInApi.determineMimeType(srvObj.getCfg(), stagingFilename) logger.debug("Determining file size ...") fileSize = ngamsPlugInApi.getFileSize(stagingFilename) return ngamsPlugInApi.genDapiSuccessStat( diskInfo.getDiskId(), relFilename, fileId, fileVersion, fformat, fileSize, uncomprSize, compression, relPath, diskInfo.getSlotId(), fileExists, complFilename) except Exception: raise Exception("ngamsNgLogPlugIn: Error handling log file: " +\ stagingFilename + ". Rejecting.")