def votable_catalogue_plugin(srv, request): mime = request.getMimeType() parDic = ngamsPlugInApi.parseDapiPlugInPars(srv.cfg, mime) options = ('host', 'user', 'password', 'dbname') for option in options: if option not in parDic: raise Exception( '%s option not specified for votable_catalogue_plugin, please provide one' % option) host, user, password, dbname = [parDic[option] for option in options] dsn = 'host=%s user=%s password=%s dbname=%s' % (host, user, password, dbname) stageFilename = request.getStagingFilename() uncomprSize = ngamsPlugInApi.getFileSize(stageFilename) file_id = os.path.basename(request.getFileUri()) diskInfo = request.getTargDiskInfo() fileVersion, relPath, relFilename, complFilename, fileExists = \ ngamsPlugInApi.genFileInfo(srv.db, srv.cfg, request, diskInfo, request.getStagingFilename(), file_id, file_id, [], []) insert_catalogue_dataproduct(dsn, file_id) return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename, file_id, fileVersion, mime, uncomprSize, uncomprSize, '', relPath, diskInfo.getSlotId(), fileExists, complFilename)
def ngamsFitsPlugIn(srvObj, reqPropsObj): """ Data Archiving Plug-In to handle archiving of FITS files. srvObj: Reference to NG/AMS Server Object (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus). """ logger.info("Plug-In handling data for file with URI: %s", os.path.basename(reqPropsObj.getFileUri())) diskInfo = reqPropsObj.getTargDiskInfo() parDic = ngamsPlugInApi.parseDapiPlugInPars(srvObj.getCfg(), reqPropsObj.getMimeType()) # Check file (size + checksum) + extract information. dpId, dateDirName, comprExt = prepFile(reqPropsObj, parDic) # Get various information about the file being handled. dpIdInfo = getDpIdInfo(reqPropsObj.getStagingFilename()) dpId = dpIdInfo[1] dateDirName = dpIdInfo[2] fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, reqPropsObj.getStagingFilename(), dpId, dpId, [dateDirName], [comprExt]) # If a compression application is specified, apply this. uncomprSize, archFileSize, mime, compression, crc = compress( reqPropsObj, parDic) # Generate status + return. logger.debug( "DAPI finished processing of file - returning to main application") return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename, dpId, fileVersion, mime, archFileSize, uncomprSize, compression, relPath, diskInfo.getSlotId(), fileExists, complFilename, crc)
def ngamsMWAFitsPlugIn(srvObj, reqPropsObj): logger.info("ngamsMWAFitsPlugIn handling data for file with URI: %s", os.path.basename(reqPropsObj.getFileUri())) mime = reqPropsObj.getMimeType() parDic = ngamsPlugInApi.parseDapiPlugInPars(srvObj.getCfg(), mime) with DBPool.db_lock: if DBPool.db_pool is None: try: dbhost = parDic['dbhost'] dbname = parDic['dbname'] dbuser = parDic['dbuser'] dbpass = parDic['dbpassword'] except Exception as e: raise Exception( 'Must specify parameters dbhost, dbname, dbuser and dbpassword' ) maxconn = 8 try: maxconn = parDic['dbmaxconn'] except: pass logger.info("ngamsMWAFitsPlugIn creating database pool") DBPool.db_pool = psycopg2.pool.ThreadedConnectionPool( minconn=1, maxconn=maxconn, host=dbhost, user=dbuser, database=dbname, password=dbpass, port=5432) diskInfo = reqPropsObj.getTargDiskInfo() stageFilename = reqPropsObj.getStagingFilename() uncomprSize = ngamsPlugInApi.getFileSize(stageFilename) fileName = os.path.basename(reqPropsObj.getFileUri()) dpId = fileName obsid, obstime, box = split_file(fileName) fileVersion, relPath, relFilename, complFilename, fileExists = \ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, reqPropsObj.getStagingFilename(), dpId, dpId, [], []) logger.info("ngamsMWAFitsPlugIn version: %s path: %s relfilename: %s" % (fileVersion, relPath, relFilename)) sql = ( 'INSERT INTO data_files (observation_num, filetype, size, filename, site_path, host)' ' VALUES (%s, %s, %s, %s, %s, %s)') uri = 'http://mwangas/RETRIEVE?file_id=%s' % (fileName) logger.info('Inserting: %s', sql % (obsid, 8, uncomprSize, fileName, uri, box)) conn = None try: conn = DBPool.db_pool.getconn() conn.set_isolation_level( psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED) with conn.cursor() as cur: try: cur.execute(sql, [obsid, 8, uncomprSize, fileName, uri, box]) except Exception as e: ''' Do not want to raise an exception here because the sky data is more important than the metadata i.e. we want to keep the sky data even if there is a database problem. If there is an issue with an INSERT then go through and add it manually later (very unlikely to happen)''' conn.rollback() logger.error('Insert error: %s Message: %s', sql % (obsid, 8, uncomprSize, fileName, uri, box), str(e)) else: conn.commit() except Exception: logger.exception('Insert error: %s', sql % (obsid, 8, uncomprSize, fileName, uri, box)) finally: if conn: DBPool.db_pool.putconn(conn=conn) return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename, dpId, fileVersion, mime, uncomprSize, uncomprSize, '', relPath, diskInfo.getSlotId(), fileExists, complFilename)
def modifies_content(srvObj, reqPropsObj): parDic = ngamsPlugInApi.parseDapiPlugInPars(srvObj.getCfg(), reqPropsObj.getMimeType()) return _compress_data(parDic)