def votable_catalogue_plugin(srv, request):
mime = request.getMimeType()
parDic = ngamsPlugInApi.parseDapiPlugInPars(srv.cfg, mime)
options = ('host', 'user', 'password', 'dbname')
for option in options:
if option not in parDic:
raise Exception(
'%s option not specified for votable_catalogue_plugin, please provide one'
% option)
host, user, password, dbname = [parDic[option] for option in options]
dsn = 'host=%s user=%s password=%s dbname=%s' % (host, user, password,
dbname)
stageFilename = request.getStagingFilename()
uncomprSize = ngamsPlugInApi.getFileSize(stageFilename)
file_id = os.path.basename(request.getFileUri())
diskInfo = request.getTargDiskInfo()
fileVersion, relPath, relFilename, complFilename, fileExists = \
ngamsPlugInApi.genFileInfo(srv.db, srv.cfg, request, diskInfo,
request.getStagingFilename(), file_id, file_id,
[], [])
insert_catalogue_dataproduct(dsn, file_id)
return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename,
file_id, fileVersion, mime,
uncomprSize,
uncomprSize, '', relPath,
diskInfo.getSlotId(), fileExists,
complFilename)
def ngamsFitsPlugIn(srvObj, reqPropsObj):
"""
Data Archiving Plug-In to handle archiving of FITS files.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Standard NG/AMS Data Archiving Plug-In Status as generated
by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus).
"""
logger.info("Plug-In handling data for file with URI: %s",
os.path.basename(reqPropsObj.getFileUri()))
diskInfo = reqPropsObj.getTargDiskInfo()
parDic = ngamsPlugInApi.parseDapiPlugInPars(srvObj.getCfg(),
reqPropsObj.getMimeType())
# Check file (size + checksum) + extract information.
dpId, dateDirName, comprExt = prepFile(reqPropsObj, parDic)
# Get various information about the file being handled.
dpIdInfo = getDpIdInfo(reqPropsObj.getStagingFilename())
dpId = dpIdInfo[1]
dateDirName = dpIdInfo[2]
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(),
reqPropsObj, diskInfo,
reqPropsObj.getStagingFilename(),
dpId, dpId, [dateDirName],
[comprExt])
# If a compression application is specified, apply this.
uncomprSize, archFileSize, mime, compression, crc = compress(
reqPropsObj, parDic)
# Generate status + return.
logger.debug(
"DAPI finished processing of file - returning to main application")
return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename,
dpId, fileVersion, mime,
archFileSize, uncomprSize,
compression, relPath,
diskInfo.getSlotId(), fileExists,
complFilename, crc)
def ngamsMWAFitsPlugIn(srvObj, reqPropsObj):
logger.info("ngamsMWAFitsPlugIn handling data for file with URI: %s",
os.path.basename(reqPropsObj.getFileUri()))
mime = reqPropsObj.getMimeType()
parDic = ngamsPlugInApi.parseDapiPlugInPars(srvObj.getCfg(), mime)
with DBPool.db_lock:
if DBPool.db_pool is None:
try:
dbhost = parDic['dbhost']
dbname = parDic['dbname']
dbuser = parDic['dbuser']
dbpass = parDic['dbpassword']
except Exception as e:
raise Exception(
'Must specify parameters dbhost, dbname, dbuser and dbpassword'
)
maxconn = 8
try:
maxconn = parDic['dbmaxconn']
except:
pass
logger.info("ngamsMWAFitsPlugIn creating database pool")
DBPool.db_pool = psycopg2.pool.ThreadedConnectionPool(
minconn=1,
maxconn=maxconn,
host=dbhost,
user=dbuser,
database=dbname,
password=dbpass,
port=5432)
diskInfo = reqPropsObj.getTargDiskInfo()
stageFilename = reqPropsObj.getStagingFilename()
uncomprSize = ngamsPlugInApi.getFileSize(stageFilename)
fileName = os.path.basename(reqPropsObj.getFileUri())
dpId = fileName
obsid, obstime, box = split_file(fileName)
fileVersion, relPath, relFilename, complFilename, fileExists = \
ngamsPlugInApi.genFileInfo(srvObj.getDb(),
srvObj.getCfg(),
reqPropsObj,
diskInfo,
reqPropsObj.getStagingFilename(),
dpId,
dpId,
[],
[])
logger.info("ngamsMWAFitsPlugIn version: %s path: %s relfilename: %s" %
(fileVersion, relPath, relFilename))
sql = (
'INSERT INTO data_files (observation_num, filetype, size, filename, site_path, host)'
' VALUES (%s, %s, %s, %s, %s, %s)')
uri = 'http://mwangas/RETRIEVE?file_id=%s' % (fileName)
logger.info('Inserting: %s',
sql % (obsid, 8, uncomprSize, fileName, uri, box))
conn = None
try:
conn = DBPool.db_pool.getconn()
conn.set_isolation_level(
psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED)
with conn.cursor() as cur:
try:
cur.execute(sql, [obsid, 8, uncomprSize, fileName, uri, box])
except Exception as e:
''' Do not want to raise an exception here because the sky data is more important than the metadata
i.e. we want to keep the sky data even if there is a database problem.
If there is an issue with an INSERT then go through and add it manually later (very unlikely to happen)'''
conn.rollback()
logger.error('Insert error: %s Message: %s',
sql % (obsid, 8, uncomprSize, fileName, uri, box),
str(e))
else:
conn.commit()
except Exception:
logger.exception('Insert error: %s',
sql % (obsid, 8, uncomprSize, fileName, uri, box))
finally:
if conn:
DBPool.db_pool.putconn(conn=conn)
return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename,
dpId, fileVersion, mime,
uncomprSize,
uncomprSize, '', relPath,
diskInfo.getSlotId(), fileExists,
complFilename)
def modifies_content(srvObj, reqPropsObj):
parDic = ngamsPlugInApi.parseDapiPlugInPars(srvObj.getCfg(),
reqPropsObj.getMimeType())
return _compress_data(parDic)