def syncCachesCheckFiles(srvObj,
filenames = []):
"""
Synchronize the file caches and check that the given files are accessible.
If problems are encountered, an exception is raised indicating which
file cannot be accessed.
srvObj: Server object (ngamsServer).
filenames: Filelist to check (list).
Returns: Void.
"""
try:
diskSyncPlugIn = srvObj.getCfg().getDiskSyncPlugIn()
if (diskSyncPlugIn):
logger.debug("Invoking Disk Sync Plug-In: %s ...", diskSyncPlugIn)
plugInMethod = loadPlugInEntryPoint(diskSyncPlugIn)
plugInMethod(srvObj)
logger.debug("Invoked Disk Sync Plug-In: %s", diskSyncPlugIn)
else:
logger.warning("No Disk Sync Plug-In defined - consider to provide one!")
#commands.getstatusoutput("sync")
for file in filenames: os.stat(file)
except Exception as e:
errMsg = "Severe error occurred! Could not sync file caches or " +\
"file not accessible! Error: " + str(e)
raise Exception(errMsg)
def wakeUpHost(srvObj, suspHost):
"""
Wake up a host which has been suspended. After invoking the appropriate
Wake-Up Plug-In it is checked within the time-out defined in the
NG/AMS Configuration File if the server, which was woken up is up
and running. If this is not the case within the specified time-out,
an exception is thrown.
srvObj: Reference to instance of the NG/AMS server (ngamsServer).
suspHost: Host name of the suspended host (string).
Returns: Void.
"""
T = TRACE()
wakeUpPi = srvObj.getCfg().getWakeUpPlugIn()
portNo = srvObj.getDb().getPortNoFromHostId(suspHost)
try:
plugInMethod = loadPlugInEntryPoint(wakeUpPi)
plugInMethod(srvObj, suspHost)
ipAddress = srvObj.getDb().getIpFromHostId(suspHost)
ngamsHighLevelLib.pingServer(ipAddress, portNo,
srvObj.getCfg().getWakeUpCallTimeOut())
except Exception:
logger.exception("Error waking up host %s", suspHost)
raise
def getDiskInfo(srvObj,
reqPropsObj = None):
"""
Invokes the plug-in listed in the configuration to extract the information
about the disks mounted. The information is returned in a dictionary which
contains the Slot IDs as keys, and elements which each is an object of type
ngamsPhysDiskInfo.
srvObj: Reference to NG/AMS Server class instance (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Dictionary containing information about the
disks (dictionary).
"""
plugIn = srvObj.getCfg().getOnlinePlugIn()
logger.info("Invoking System Online Plug-In: %s(srvObj)", plugIn)
plugInMethod = loadPlugInEntryPoint(plugIn)
diskInfoDic = plugInMethod(srvObj, reqPropsObj)
if not diskInfoDic:
if (not ngamsLib.trueArchiveProxySrv(srvObj.getCfg())):
errMsg = genLog("NGAMS_NOTICE_NO_DISKS_AVAIL",
[srvObj.getHostId(),
srvObj.getHostId()])
logger.warning(errMsg)
logger.debug("Disk Dictionary: %s", str(diskInfoDic))
return diskInfoDic
def performStaging(srvObj, reqPropsObj, httpRef, filename):
"""
if the staging plugin is set, then perform staging
using the registered staging plugin
if the file is offline (i.e. on Tape)
srvObj: Reference to NG/AMS server class object (ngamsServer).
filename: File to be processed (string).
"""
if srvObj.getCfg().getFileStagingEnable() != 1:
return
fspi = srvObj.getCfg().getFileStagingPlugIn()
if not fspi:
return
logger.info("Invoking FSPI.isFileOffline: %s to check file: %s", fspi,
filename)
isFileOffline = loadPlugInEntryPoint(fspi, 'isFileOffline')
if isFileOffline(filename) == 0:
return
logger.info("Invoking FSPI.stageFiles: %s to check file: %s", fspi,
filename)
stageFiles = loadPlugInEntryPoint(fspi, 'stageFiles')
try:
st = time.time()
stageFiles(filenames=[filename],
requestObj=reqPropsObj,
serverObj=srvObj)
howlong = time.time() - st
fileSize = getFileSize(filename)
logger.debug('Staging rate = %.0f Bytes/s (%.0f seconds) for file %s',
fileSize / howlong, howlong, filename)
except socket.timeout:
errMsg = 'Staging timed out: %s' % filename
logger.warning(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=504)
raise
def get_plugins(srvObj):
"""
Returns the list of plug-ins that need to be run in this janitor process
"""
built_in = [
'backlog_buffer_checker', 'old_requests_cleaner', 'expired_data_cleaner',
'notifications_sender', 'rotated_logfiles_handler', 'disk_space_checker',
'wake_up_request_processor', 'host_suspender'
]
built_in = [importlib.import_module('.' + bi, __package__).run for bi in built_in]
user_plugins = srvObj.getCfg().getJanitorPlugins()
user_plugins = [loadPlugInEntryPoint(p, entryPointMethodName='run') for p in user_plugins]
return built_in + user_plugins
def get_logfile_handler_plugins(cfg):
global _lh_plugins
if _lh_plugins is None:
_lh_plugins = []
try:
for name, pars in cfg.logfile_handler_plugins:
func = loadPlugInEntryPoint(name, 'run')
pars = ngamsLib.parseRawPlugInPars(pars)
_lh_plugins.append(functools.partial(func, pars))
except:
# Only report the error, but nothing else
logger.exception("Error while loading logfile handler plug-ins,"
"no extra functionality available")
return _lh_plugins
def get_plugins(srvObj):
"""
Returns the list of plug-ins that need to be run in this janitor process
"""
hardcoded = [
'janitor.backlog_buffer_checker', 'janitor.old_requests_cleaner',
'janitor.expired_data_cleaner', 'janitor.notifications_sender',
'janitor.rotated_logfiles_handler', 'janitor.disk_space_checker',
'janitor.wake_up_request_processor', 'janitor.host_suspender'
]
user_plugins = srvObj.getCfg().getJanitorPlugins()
return [
loadPlugInEntryPoint(n, entryPointMethodName='run')
for n in hardcoded + user_plugins
]
def printLabel(srvObj, diskId, hostId, slotId, reqPropsObj=None):
"""
Print a label by invoking the label printer.
srvObj: Reference to instance of NG/AMS Server class (ngamsServer).
diskId: ID of disk to print label for (string).
hostId: Host ID of host hosting the disk (string).
slotId: Slot ID of slot in which the disk is inserted (string).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
Returns: Void.
"""
T = TRACE()
label = srvObj.getDb().getLogicalNameFromDiskId(diskId)
if (not label):
errMsg = "Empty Logical Name returned for Disk ID: %s" % diskId
raise Exception(errMsg)
logger.debug(
"Generating label for disk with ID: %s - Label text (Logical Name): %s",
diskId, label)
plugIn = srvObj.getCfg().getLabelPrinterPlugIn()
prStr = label + " " + hostId + ":" + slotId
logger.debug("Invoking Label Printer Plug-In: %s(srvObj, %s)", plugIn,
prStr)
global _labelPrinterSem
_labelPrinterSem.acquire()
try:
pluginMethod = loadPlugInEntryPoint(plugIn)
pluginMethod(srvObj, prStr, reqPropsObj)
except:
_labelPrinterSem.release()
raise
_labelPrinterSem.release()
logger.info(
"Generated label for disk with ID: %s - Label text (Logical Name): %s",
diskId, label)
def performProcessing(srvObj, reqPropsObj, filename, mimeType):
"""
Carry out the processing requested.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
filename: File to be processed (string).
mimeType: Mime-type of file (string).
Returns: List with ngamsDppiStatus object
(list/ngamsDppiStatus objects).
"""
T = TRACE()
statusObjList = []
# Carry out the processing specified. If no processing is
# specified, we simply set the source file as the file to be send.
if (reqPropsObj.hasHttpPar("processing")):
dppi = reqPropsObj.getHttpPar("processing")
# Before starting to process, check if the specified DPPI
# is supported by this NG/AMS.
if dppi not in srvObj.getCfg().dppi_plugins:
errMsg = genLog("NGAMS_ER_ILL_DPPI", [dppi])
raise Exception, errMsg
# Invoke the DPPI.
logger.info("Invoking DPPI: %s to process file: %s", dppi, filename)
plugInMethod = loadPlugInEntryPoint(dppi)
statusObj = plugInMethod(srvObj, reqPropsObj, filename)
else:
logger.info("No processing requested - sending back file as is")
resultObj = ngamsDppiStatus.ngamsDppiResult(NGAMS_PROC_FILE, mimeType,
filename, filename)
statusObj = ngamsDppiStatus.ngamsDppiStatus().addResult(resultObj)
statusObjList.append(statusObj)
return statusObjList
def run(srvObj, stopEvt):
hostId = srvObj.getHostId()
cfg = srvObj.getCfg()
now = time.time()
suspend = (cfg.getIdleSuspension() and
not srvObj.getHandlingCmd() and
not srvObj.getDb().getSrvDataChecking(hostId) and
now - srvObj.getLastReqEndTime() >= cfg.getIdleSuspensionTime())
if not suspend:
return
# Conditions are met for suspending this NGAS host.
logger.info("NG/AMS Server %s suspending itself", hostId)
# If Data Checking is on, we request a wake-up call.
if cfg.getDataCheckActive():
wakeUpSrv = cfg.getWakeUpServerHost()
nextDataCheck = srvObj.getNextDataCheckTime()
srvObj.reqWakeUpCall(wakeUpSrv, nextDataCheck)
# Now, suspend this host.
srvObj.getDb().markHostSuspended(hostId)
suspPi = cfg.getSuspensionPlugIn()
logger.debug("Invoking Suspension Plug-In: %s to " +\
"suspend NG/AMS Server: %s", suspPi, hostId)
try:
plugInMethod = loadPlugInEntryPoint(suspPi)
plugInMethod(srvObj)
except Exception, e:
errMsg = "Error suspending server with Suspension Plug-In %s"
logger.exception(errMsg, suspPi)
ngamsNotification.notify(hostId, cfg, NGAMS_NOTIF_ERROR,
"ERROR INVOKING SUSPENSION "+\
"PLUG-IN", errMsg % (suspPi,) + ": %s" % (str(e),))
raise
def handleOffline(srvObj, reqPropsObj=None):
"""
Carry out the actions to put the system in Offline state (Standby).
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
stopJanitorThr: Setting this to 0, the function will not try to
stop the Janitor Thread. The reason for this is that
the Janitor Thread may bring the server to Offline
(integer/0|1).
Returns: Void.
"""
# Stop/delete Janitor Thread + Data Check Thread + inform other
# possible threads to stop execution (if running).
srvObj.stopJanitorThread()
srvObj.stopDataCheckThread()
ngamsSubscriptionThread.stopSubscriptionThread(srvObj)
srvObj.stopUserServiceThread()
srvObj.stopMirControlThread()
srvObj.stopCacheControlThread()
srvObj.setThreadRunPermission(0)
if srvObj.getCfg().getSubscrEnable():
srvObj.remote_subscription_creation_evt.set()
logger.debug("Prepare NG/AMS for Offline State ...")
# Unsubscribe possible subscriptions. This is tried only once.
if (srvObj.getCfg().getAutoUnsubscribe()):
for subscrObj in srvObj.getSubscrStatusList():
subs_host, subs_port = subscrObj.getHostId(), subscrObj.getPortNo()
try:
resp = ngamsHttpUtils.httpGet(
subs_host, subs_port, NGAMS_UNSUBSCRIBE_CMD,
[["url", subscrObj.getId()]])
with contextlib.closing(resp):
stat = ngamsStatus.to_status(
resp, "%s:%d" % (subs_host, subs_port),
NGAMS_UNSUBSCRIBE_CMD)
if stat.getStatus() != NGAMS_SUCCESS:
logger.error("Error when unsubscribing from %s:%d: %s",
subs_host, subs_port, stat.getMessage())
except Exception:
msg = "Problem occurred while cancelling subscription " +\
"(host/port): %s/%d. Subscriber ID: %s"
logger.exception(msg, subscrObj.getHostId(),
subscrObj.getPortNo(), subscrObj.getId())
srvObj.resetSubscrStatusList()
# Check if there are files located in the Staging Areas of the
# Disks. In case yes, move these to the Back-Log Area to have them
# treated at a later stage.
checkStagingAreas(srvObj)
# Dump disk info on all disks, invoke the Offline Plug-In to prepare the
# disks for offline, and mark the disks as unmounted in the DB.
ngamsDiskUtils.dumpDiskInfoAllDisks(srvObj.getHostId(), srvObj.getDb(),
srvObj.getCfg())
plugIn = srvObj.getCfg().getOfflinePlugIn()
if (srvObj.getCfg().getSimulation() == 0):
logger.info("Invoking System Offline Plug-In: %s(srvObj, reqPropsObj)",
plugIn)
plugInMethod = loadPlugInEntryPoint(plugIn)
plugInRes = plugInMethod(srvObj, reqPropsObj)
else:
pass
ngamsDiskUtils.markDisksAsUnmountedInDb(srvObj.getHostId(), srvObj.getDb(),
srvObj.getCfg())
# Send out possible Retained Email Notification Messages.
flushMsg = "NOTE: Distribution of retained Email Notification Messages " +\
"forced at Offline"
ngamsNotification.checkNotifRetBuf(srvObj.getHostId(), srvObj.getCfg(), 1,
flushMsg)
logger.info("NG/AMS prepared for Offline State")
def getDiskInfo(srvObj, reqPropsObj=None):
"""
Invokes the plug-in listed in the configuration to extract the information
about the disks mounted. The information is returned in a dictionary which
contains the Slot IDs as keys, and elements which each is an object of type
ngamsPhysDiskInfo.
srvObj: Reference to NG/AMS Server class instance (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Dictionary containing information about the
disks (dictionary).
"""
T = TRACE()
diskInfoDic = {}
plugIn = srvObj.getCfg().getOnlinePlugIn()
if (not srvObj.getCfg().getSimulation()):
logger.info("Invoking System Online Plug-In: %s(srvObj)", plugIn)
plugInMethod = loadPlugInEntryPoint(plugIn)
diskInfoDic = plugInMethod(srvObj, reqPropsObj)
if (len(diskInfoDic) == 0):
if (not ngamsLib.trueArchiveProxySrv(srvObj.getCfg())):
errMsg = genLog("NGAMS_NOTICE_NO_DISKS_AVAIL",
[srvObj.getHostId(),
srvObj.getHostId()])
logger.warning(errMsg)
else:
if (srvObj.getCfg().getAllowArchiveReq()):
logger.debug("Running as a simulated archiving system - generating " +\
"simulated disk info ...")
portNo = 0
slotIdLst = srvObj.getCfg().getSlotIds()
for slotId in slotIdLst:
if (slotId != ""):
storageSet = srvObj.getCfg().\
getStorageSetFromSlotId(slotId)
if (storageSet.getMainDiskSlotId() == slotId):
diskType = "Main"
else:
diskType = "Rep"
mtRootDir = srvObj.getCfg().getRootDirectory()
mtPt = os.path.normpath(mtRootDir +\
"/"+storageSet.getStorageSetId() +\
"-" + diskType + "-" + str(slotId))
diskId = re.sub("/", "-", mtPt)[1:]
serialNo = "SERIAL-NUMBER-" + str(portNo)
manufact = "TEST-MANUFACTURER"
diskInfoDic[slotId] = ngamsPhysDiskInfo.ngamsPhysDiskInfo()
diskInfoDic[slotId].\
setPortNo(portNo).\
setSlotId(slotId).\
setMountPoint(mtPt).\
setStatus("OK").\
setCapacityGb(100).\
setModel("TEST-MODEL").\
setSerialNo(serialNo).\
setType("TEST-TYPE").\
setManufacturer(manufact).\
setDiskId(diskId).\
setDeviceName("/dev/dummy" + slotId)
# We create the mount directory if it does not exist
checkCreatePath(mtPt)
portNo = portNo + 1
else:
# It is not an Archiving System. We check which of the directories
# in the Ngas.RootDirectory contain an NgasDiskInfo file.
# These are considered as available disks in simulation mode.
baseDir = srvObj.getCfg().getRootDirectory()
dirList = glob.glob(os.path.normpath(baseDir + "/*"))
for mtPt in dirList:
checkPath = os.path.normpath(mtPt + "/" + NGAMS_DISK_INFO)
logger.debug("Checking if path exists: %s", checkPath)
if (os.path.exists(checkPath)):
slotId = string.split(mtPt, "-")[-1]
portNo = (int(slotId) - 1)
diskId = re.sub("/", "-", mtPt)
serialNo = "SERIAL-NUMBER-" + str(portNo)
diskInfoDic[slotId]=ngamsPhysDiskInfo.ngamsPhysDiskInfo().\
setPortNo(portNo).\
setSlotId(slotId).\
setMountPoint(mtPt).\
setStatus("OK").\
setCapacityGb(100).\
setModel("TEST-MODEL").\
setSerialNo(serialNo).\
setType("TEST-TYPE").\
setManufacturer("TEST-MANUFACTURER").\
setDiskId(diskId).\
setDeviceName("/dev/dummy" + slotId)
logger.debug("Disk Dictionary: %s", str(diskInfoDic))
return diskInfoDic
def archiveFromFile(srvObj,
filename,
noReplication=0,
mimeType=None,
reqPropsObj=None):
"""
Archive a file directly from a file as source.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
filename: Name of file to archive (string).
noReplication: Flag to enable/disable replication (integer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
Returns: Execution result object of DAPI
"""
T = TRACE()
logger.debug("Archiving file: %s", filename)
logger.debug("Mimetype used is %s", mimeType)
if (reqPropsObj):
logger.debug("Request Properties Object given - using this")
reqPropsObjLoc = reqPropsObj
else:
logger.debug("No Request Properties Object given - creating one")
reqPropsObjLoc = ngamsArchiveUtils.ngamsReqProps.ngamsReqProps()
stagingFile = filename
try:
if (mimeType == None):
mimeType = ngamsHighLevelLib.determineMimeType(
srvObj.getCfg(), filename)
archive_start = time.time()
# Prepare dummy ngamsReqProps object (if an object was not given).
if (not reqPropsObj):
reqPropsObjLoc.setMimeType(mimeType)
reqPropsObjLoc.setStagingFilename(filename)
reqPropsObjLoc.setHttpMethod(NGAMS_HTTP_GET)
reqPropsObjLoc.setCmd(NGAMS_ARCHIVE_CMD)
reqPropsObjLoc.setSize(os.path.getsize(filename))
reqPropsObjLoc.setFileUri(NGAMS_HTTP_FILE_URL + filename)
reqPropsObjLoc.setNoReplication(noReplication)
# If no target disk is defined, find one suitable disk.
if (not reqPropsObjLoc.getTargDiskInfo()):
try:
trgDiskInfo = ngamsArchiveUtils.ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
mimeType, 0,
reqSpace=reqPropsObjLoc.getSize())
reqPropsObjLoc.setTargDiskInfo(trgDiskInfo)
# copy the file to the staging area of the target disk
stagingFile = trgDiskInfo.getMountPoint(
) + '/staging/' + os.path.basename(filename)
cpFile(filename, stagingFile)
reqPropsObjLoc.setStagingFilename(stagingFile)
except Exception as e:
errMsg = str(e) + ". Attempting to archive local file: " +\
filename
ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_NO_DISKS,
"NO DISKS AVAILABLE", errMsg)
raise Exception(errMsg)
# Set the log cache to 1 during the handling of the file.
plugIn = srvObj.getMimeTypeDic()[mimeType]
logger.debug("Invoking DAPI: %s to handle file: %s", plugIn,
stagingFile)
plugInMethod = loadPlugInEntryPoint(plugIn)
resMain = plugInMethod(srvObj, reqPropsObjLoc)
# Move the file to final destination.
st = time.time()
mvFile(reqPropsObjLoc.getStagingFilename(),
resMain.getCompleteFilename())
iorate = reqPropsObjLoc.getSize() / (time.time() - st)
ngamsArchiveUtils.postFileRecepHandling(srvObj, reqPropsObjLoc,
resMain, trgDiskInfo)
except Exception as e:
# If another error occurs, than one qualifying for Back-Log
# Buffering the file, we have to log an error.
if (ngamsHighLevelLib.performBackLogBuffering(srvObj.getCfg(),
reqPropsObjLoc, e)):
logger.exception(
"Tried to archive local file %s, keeping original file",
filename)
return [NGAMS_FAILURE, str(e), NGAMS_FAILURE]
else:
logger.exception("Tried to archive local file %s, " + \
"moving it to Bad Files Directory " + \
"-- cannot be handled", filename)
ngamsHighLevelLib.moveFile2BadDir(srvObj.getCfg(), filename)
# Remove pickle file if available.
pickleObjFile = filename + "." + NGAMS_PICKLE_FILE_EXT
if (os.path.exists(pickleObjFile)):
logger.debug("Removing Back-Log Buffer Pickle File: %s",
pickleObjFile)
rmFile(pickleObjFile)
return [NGAMS_FAILURE, str(e), NGAMS_FAILURE]
# If the file was handled successfully, we remove it from the
# Back-Log Buffer Directory unless the local file was a log-file
# in which case we leave the cleanup to the Janitor-Thread.
if stagingFile.find('LOG-ROTATE') > -1:
logger.debug("Successfully archived local file: %s", filename)
else:
logger.debug(
"Successfully archived local file: %s. Removing staging file.",
filename)
rmFile(stagingFile)
rmFile(stagingFile + "." + NGAMS_PICKLE_FILE_EXT)
logger.debug("Archived local file: %s. Time (s): %.3f", filename,
time.time() - archive_start)
return (resMain, trgDiskInfo, iorate)
def _registerExec(srvObj,
fileListDbmName,
tmpFilePat,
diskInfoDic,
reqPropsObj=None):
"""
Register the files listed in the File List DBM (ngamsDbm), which
match the mime-type(s) either specified in the 'mimeType' parameter,
or if this is not specified, which match all the mime-types specified
in the configuration file.
When the registration procedure has been executed, the function sends
an Email Notification message indicating which files were registered
if the HTTP parameter 'notif_email' is given.
The functions creates a File Info Objects per file handled and
writes this in a temporary file, which is a DBM file. The
keys in this DB is simply the file number in the sequence of files
handled, pointing to a pickled ngamsFileInfo object.
Each of the File Info Objects indicates if the file was registered or not.
This is done by setting the tag of the File Info Object to one of the
following values:
REGISTERED: The file was successfully registered in the NGAS DB
FAILED: The file was selected for registration but could not
be properly registered because of inconsistencies.
The status will be of the format: 'FAILED[: <reason>]'.
REJECTED: A file found under the specified path directory was
not accepted for cloning, usually because the mime-type
was not correct. The status will be of the format:
'REJECTED[: <reason>]'.
Note, that registration is comparable to archiving of files. For that
reason a DAPI must be provided for each type of file that should be
registered. If this is not fullfilled, the file registration will
fail.
Only files stored on one of the NGAS disks configured in the
configuration file, are considered. Files stored in other locations
are ignored.
srvObj: Instance of NG/AMS Server object (ngamsServer).
fileListDbmName: Name of a DBM containing the information
about the files to be registered. Each element in the
list is referred to by a key, which is a number.
These points to a pickled list for each file containing
the following information:
[<Filename>, <Disk ID>, <Mime-Type>]
The information for each disk concerned is also contained
in the DB referred to by its Disk ID and by its mount
point. The data is a pickled instance of the ngamsDiskInfo
class (string).
tmpFilePat: Pattern for temporary files used during the registration
process (string).
diskInfoDic: Dictionary with Disk IDs as keys pointing to the info
about the disk (dictionary/ngamsDiskInfo).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
Returns: Void.
"""
emailNotif = 0
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")):
emailNotif = 1
# Create the temporary BSD DB to contain the information for the
# Email Notification Message.
if (emailNotif):
regDbmName = tmpFilePat + "_NOTIF_EMAIL"
regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm=1)
# Open the DBM containing the list of files to (possibly) register.
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1)
# Want to parse files in alphabetical order.
# TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting.
tmpFileList = tmpFilePat + "_FILE_LIST"
rmFile(tmpFileList)
with open(tmpFileList, "wb") as fo:
fileListDbm.initKeyPtr()
while (1):
dbmKey, fileInfo = fileListDbm.getNext()
if (not dbmKey): break
fo.write(dbmKey + b"\n")
sortFileList = tmpFilePat + "_SORT_FILE_LIST"
rmFile(sortFileList)
shellCmd = "sort %s > %s" % (tmpFileList, sortFileList)
stat, out, err = ngamsCore.execCmd(shellCmd)
if (stat != 0):
raise Exception("Error executing command: %s. Error: %s, %s" %\
(shellCmd, str(out), str(err)))
rmFile(tmpFileList)
# Go through each file in the list, check if the mime-type is among the
# ones, which apply for registration. If yes try to register the file
# by invoking the corresponding DAPI on the file.
fileRegCount = 0
fileFailCount = 0
fileRejectCount = 0
regTimeAccu = 0.0
fileCount = 0
fo = open(sortFileList)
run = 1
while (run):
reg_start = time.time()
dbmKey = fo.readline()
if (dbmKey.strip() == ""):
run = 0
continue
fileInfo = fileListDbm.get(dbmKey[0:-1])
filename = fileInfo[0]
diskId = fileInfo[1]
mimeType = fileInfo[2]
# Register the file. Check first, that exactly this file is
# not already registered. In case it is, the file will be rejected.
regPi = srvObj.getCfg().register_plugins[mimeType]
logger.debug("Plugin found for %s: %s", mimeType, regPi)
params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars)
tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\
setMimeType(mimeType).\
setStagingFilename(filename).\
setTargDiskInfo(diskInfoDic[diskId]).\
setHttpMethod(NGAMS_HTTP_GET).\
setCmd(NGAMS_REGISTER_CMD).\
setSize(os.path.getsize(filename)).\
setFileUri(filename).\
setNoReplication(1)
tmpFileObj = ngamsFileInfo.ngamsFileInfo()
try:
# Invoke Registration Plug-In.
piName = regPi.name
plugInMethod = loadPlugInEntryPoint(piName)
piRes = plugInMethod(srvObj, tmpReqPropsObj, params)
del tmpReqPropsObj
# Check if this file is already registered on this disk. In case
# yes, it is not registered again.
files = srvObj.db.getFileSummary1(srvObj.getHostId(),
[piRes.getDiskId()],
[piRes.getFileId()])
fileRegistered = 0
for tmpFileInfo in files:
tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT]
tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME]
tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\
tmpFilename)
if (tmpComplFilename == filename):
fileRegistered = 1
break
if (fileRegistered):
fileRejectCount += 1
tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\
"and path: %s is already registered on disk " +\
"with Disk ID: %s"
tmpMsg = tmpMsgForm % (piRes.getFileId(),
piRes.getFileVersion(), filename,
piRes.getDiskId())
logger.warning(tmpMsg + ". File is not registered again.")
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(tmpMsg)
regDbm.addIncKey(tmpFileObj)
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(
srvObj, reqPropsObj, regTimeAccu)
regTimeAccu += time.time() - reg_start
fileCount += 1
continue
# Calculate checksum. We maintain the old name for backwards
# compatibility
crc_variant = srvObj.cfg.getCRCVariant()
if crc_variant == ngamsFileUtils.CHECKSUM_CRC32_INCONSISTENT:
crc_variant = 'ngamsGenCrc32'
checksum = ngamsFileUtils.get_checksum(65536, filename,
crc_variant) or ''
# Move file and update information about file in the NGAS DB.
mvFile(filename, piRes.getCompleteFilename())
ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum,
crc_variant)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes)
ngamsLib.makeFileReadOnly(piRes.getCompleteFilename())
if (emailNotif):
uncomprSize = piRes.getUncomprSize()
ingestDate = time.time()
creDateSecs = getFileCreationTime(filename)
tmpFileObj.\
setDiskId(diskId).\
setFilename(filename).\
setFileId(piRes.getFileId()).\
setFileVersion(piRes.getFileVersion()).\
setFormat(piRes.getFormat()).\
setFileSize(piRes.getFileSize()).\
setUncompressedFileSize(uncomprSize).\
setCompression(piRes.getCompression()).\
setIngestionDate(ingestDate).\
setIgnore(0).\
setChecksum(checksum).\
setChecksumPlugIn(crc_variant).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDateSecs).\
setTag("REGISTERED")
fileRegCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
fileVer = fio.getFileVersion()
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, piRes.getFileId(), fileVer, filename)
# Generate a confirmation log entry.
msg = genLog("NGAMS_INFO_FILE_REGISTERED", [
filename,
piRes.getFileId(),
piRes.getFileVersion(),
piRes.getFormat()
])
time.sleep(0.005)
regTime = time.time() - reg_start
msg = msg + ". Time: %.3fs." % (regTime)
logger.info(msg, extra={'to_syslog': 1})
except Exception as e:
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)])
logger.error(errMsg)
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(errMsg)
fileFailCount += 1
regTime = time.time() - reg_start
# TODO (rtobar, 2016-01): Why don't we raise an exception here?
# Otherwise the command appears as successful on the
# client-side
# Add the file information in the registration report.
if (emailNotif): regDbm.addIncKey(tmpFileObj)
# Update request status time information.
regTimeAccu += regTime
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
fileCount += 1
fo.close()
rmFile(sortFileList)
if (emailNotif): regDbm.sync()
del fileListDbm
rmFile(fileListDbmName + "*")
# Final update of the Request Status.
if (reqPropsObj):
if (reqPropsObj.getExpectedCount() and reqPropsObj.getActualCount()):
complPercent = (100.0 * (float(reqPropsObj.getActualCount()) /
float(reqPropsObj.getExpectedCount())))
else:
complPercent = 100.0
reqPropsObj.setCompletionPercent(complPercent, 1)
reqPropsObj.setCompletionTime(1)
srvObj.updateRequestDb(reqPropsObj)
# Send Register Report with list of files cloned to a possible
# requestor(select) of this.
if (emailNotif):
xmlStat = 0
if (xmlStat):
# Create an instance of the File List Class, used to store the
# Registration Report.
regStat = ngamsFileList.\
ngamsFileList("FILE_REGISTRATION_STATUS",
"Status report for file " +\
"registration")
# Loop over the file objects in the BSD DB and add these
# in the status object.
regDbm.initKeyPtr()
while (1):
key, tmpFileObj = regDbm.getNext()
if (not key): break
regStat.addFileInfoObj(tmpFileObj)
# Set overall status of registration procedure.
regStat.setStatus("Files Found: " + str(fileCount + 1) + ", "+\
"Files Registered: " + str(fileRegCount) +\
", " +\
"Files Failed: " + str(fileFailCount) + ", " +\
"Files Rejected: " + str(fileRejectCount))
status = srvObj.genStatus(NGAMS_SUCCESS,
"REGISTER command status report").\
addFileList(regStat)
statRep = status.genXmlDoc()
statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep)
mimeType = NGAMS_XML_MT
else:
# Generate a 'simple' ASCII report.
statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
fo = open(statRep, "w")
if (reqPropsObj):
path = reqPropsObj.getHttpPar("path")
else:
path = "-----"
if (fileCount):
timePerFile = (regTimeAccu / fileCount)
else:
timePerFile = 0
tmpFormat = "REGISTER STATUS REPORT:\n\n" +\
"==Summary:\n\n" +\
"Date: %s\n" +\
"NGAS Host: %s\n" +\
"Search Path: %s\n" +\
"Total Number of Files: %d\n" +\
"Number of Registered Files: %d\n" +\
"Number of Failed Files: %d\n" +\
"Number of Rejected Files: %d\n" +\
"Total processing time (s): %.3f\n" +\
"Handling time per file (s): %.3f\n\n" +\
"==File List:\n\n"
fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), path,
fileCount, fileRegCount, fileFailCount,
fileRejectCount, regTimeAccu, timePerFile))
tmpFormat = "%-80s %-32s %-3s %-10s\n"
fo.write(tmpFormat %\
("Filename", "File ID", "Ver", "Status"))
fo.write(tmpFormat % (80 * "-", 32 * "-", 3 * "-", 10 * "-"))
regDbm.initKeyPtr()
while (1):
key, tmpFileObj = regDbm.getNext()
if (not key): break
mtPt = diskInfoDic[tmpFileObj.getDiskId()].getMountPoint()
filename = os.path.normpath(mtPt + "/" +\
tmpFileObj.getFilename())
line = tmpFormat %\
(filename, tmpFileObj.getFileId(),
str(tmpFileObj.getFileVersion()),
tmpFileObj.getTag())
fo.write(line)
fo.write(128 * "-")
fo.write("\n\n==END\n")
fo.close()
mimeType = NGAMS_TEXT_MT
# Send out the status report.
emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",")
attachmentName = "RegisterStatusReport"
if (reqPropsObj.hasHttpPar("path")):
attachmentName += "-" + reqPropsObj.getHttpPar("path").\
replace("/", "_")
ngamsNotification.notify(srvObj.host_id,
srvObj.cfg,
NGAMS_NOTIF_INFO,
"REGISTER STATUS REPORT",
statRep,
recList=emailAdrList,
force=1,
contentType=mimeType,
attachmentName=attachmentName)
del regDbm
rmFile(regDbmName + "*")
rmFile(statRep)
# Generate final status log + exit.
if (fileCount > 0):
timePerFile = (regTimeAccu / fileCount)
else:
timePerFile = 0.0
msg = "Registration procedure finished processing Register Request - " + \
"terminating. Files handled: %d. Total time: %.3fs. " + \
"Average time per file: %.3fs."
logger.debug(msg, fileCount, regTimeAccu, timePerFile)
def _registerExec(srvObj,
fileListDbmName,
tmpFilePat,
diskInfoDic,
reqPropsObj = None):
"""
Register the files listed in the File List DBM (ngamsDbm), which
match the mime-type(s) either specified in the 'mimeType' parameter,
or if this is not specified, which match all the mime-types specified
in the configuration file.
When the registration procedure has been executed, the function sends
an Email Notification message indicating which files were registered
if the HTTP parameter 'notif_email' is given.
The functions creates a File Info Objects per file handled and
writes this in a temporary file, which is a DBM file. The
keys in this DB is simply the file number in the sequence of files
handled, pointing to a pickled ngamsFileInfo object.
Each of the File Info Objects indicates if the file was registered or not.
This is done by setting the tag of the File Info Object to one of the
following values:
REGISTERED: The file was successfully registered in the NGAS DB
FAILED: The file was selected for registration but could not
be properly registered because of inconsistencies.
The status will be of the format: 'FAILED[: <reason>]'.
REJECTED: A file found under the specified path directory was
not accepted for cloning, usually because the mime-type
was not correct. The status will be of the format:
'REJECTED[: <reason>]'.
Note, that registration is comparable to archiving of files. For that
reason a DAPI must be provided for each type of file that should be
registered. If this is not fullfilled, the file registration will
fail.
Only files stored on one of the NGAS disks configured in the
configuration file, are considered. Files stored in other locations
are ignored.
srvObj: Instance of NG/AMS Server object (ngamsServer).
fileListDbmName: Name of a DBM containing the information
about the files to be registered. Each element in the
list is referred to by a key, which is a number.
These points to a pickled list for each file containing
the following information:
[<Filename>, <Disk ID>, <Mime-Type>]
The information for each disk concerned is also contained
in the DB referred to by its Disk ID and by its mount
point. The data is a pickled instance of the ngamsDiskInfo
class (string).
tmpFilePat: Pattern for temporary files used during the registration
process (string).
diskInfoDic: Dictionary with Disk IDs as keys pointing to the info
about the disk (dictionary/ngamsDiskInfo).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
Returns: Void.
"""
T = TRACE()
emailNotif = 0
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")):
emailNotif = 1
# Create the temporary BSD DB to contain the information for the
# Email Notification Message.
if (emailNotif):
regDbmName = tmpFilePat + "_NOTIF_EMAIL"
regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm = 1)
# Open the DBM containing the list of files to (possibly) register.
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm = 1)
# Want to parse files in alphabetical order.
# TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting.
tmpFileList = tmpFilePat + "_FILE_LIST"
rmFile(tmpFileList)
fo = open(tmpFileList, "w")
fileListDbm.initKeyPtr()
while (1):
dbmKey, fileInfo = fileListDbm.getNext()
if (not dbmKey): break
fo.write(dbmKey + "\n")
fo.close()
sortFileList = tmpFilePat + "_SORT_FILE_LIST"
rmFile(sortFileList)
shellCmd = "sort %s > %s" % (tmpFileList, sortFileList)
stat, out = commands.getstatusoutput(shellCmd)
if (stat != 0):
raise Exception, "Error executing command: %s. Error: %s" %\
(shellCmd, str(out))
rmFile(tmpFileList)
# Go through each file in the list, check if the mime-type is among the
# ones, which apply for registration. If yes try to register the file
# by invoking the corresponding DAPI on the file.
checksumPlugIn = srvObj.getCfg().getChecksumPlugIn().strip()
fileRegCount = 0
fileFailCount = 0
fileRejectCount = 0
regTimeAccu = 0.0
fileCount = 0
fo = open(sortFileList)
run = 1
while (run):
reg_start = time.time()
dbmKey = fo.readline()
if (dbmKey.strip() == ""):
run = 0
continue
fileInfo = fileListDbm.get(dbmKey[0:-1])
filename = fileInfo[0]
diskId = fileInfo[1]
mimeType = fileInfo[2]
# Register the file. Check first, that exactly this file is
# not already registered. In case it is, the file will be rejected.
regPi = srvObj.getCfg().register_plugins[mimeType]
logger.debug("Plugin found for %s: %s", mimeType, regPi)
params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars)
tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\
setMimeType(mimeType).\
setStagingFilename(filename).\
setTargDiskInfo(diskInfoDic[diskId]).\
setHttpMethod(NGAMS_HTTP_GET).\
setCmd(NGAMS_REGISTER_CMD).\
setSize(os.path.getsize(filename)).\
setFileUri(filename).\
setNoReplication(1)
tmpFileObj = ngamsFileInfo.ngamsFileInfo()
try:
# Invoke Registration Plug-In.
piName = regPi.name
plugInMethod = loadPlugInEntryPoint(piName)
piRes = plugInMethod(srvObj, tmpReqPropsObj, params)
del tmpReqPropsObj
# Check if this file is already registered on this disk. In case
# yes, it is not registered again.
files = srvObj.db.getFileSummary1(srvObj.getHostId(), [piRes.getDiskId()],
[piRes.getFileId()])
fileRegistered = 0
for tmpFileInfo in files:
tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT]
tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME]
tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\
tmpFilename)
if (tmpComplFilename == filename):
fileRegistered = 1
break
if (fileRegistered):
fileRejectCount += 1
tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\
"and path: %s is already registered on disk " +\
"with Disk ID: %s"
tmpMsg = tmpMsgForm % (piRes.getFileId(),
piRes.getFileVersion(), filename,
piRes.getDiskId())
logger.warning(tmpMsg + ". File is not registered again.")
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(tmpMsg)
regDbm.addIncKey(tmpFileObj)
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
regTimeAccu += time.time() - reg_start
fileCount += 1
continue
# Calculate checksum (if plug-in specified).
if (checksumPlugIn != ""):
logger.debug("Invoking Checksum Plug-In: %s to handle file: %s",
checksumPlugIn, filename)
plugInMethod = loadPlugInEntryPoint(checksumPlugIn)
checksum = plugInMethod(srvObj, filename, 0)
else:
checksum = ""
# Move file and update information about file in the NGAS DB.
mvFile(filename, piRes.getCompleteFilename())
ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum,
checksumPlugIn)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes)
ngamsLib.makeFileReadOnly(piRes.getCompleteFilename())
if (emailNotif):
uncomprSize = piRes.getUncomprSize()
ingestDate = time.time()
creDateSecs = getFileCreationTime(filename)
tmpFileObj.\
setDiskId(diskId).\
setFilename(filename).\
setFileId(piRes.getFileId()).\
setFileVersion(piRes.getFileVersion()).\
setFormat(piRes.getFormat()).\
setFileSize(piRes.getFileSize()).\
setUncompressedFileSize(uncomprSize).\
setCompression(piRes.getCompression()).\
setIngestionDate(ingestDate).\
setIgnore(0).\
setChecksum(checksum).\
setChecksumPlugIn(checksumPlugIn).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDateSecs).\
setTag("REGISTERED")
fileRegCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
fileVer = fio.getFileVersion()
ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId,
piRes.getFileId(),
fileVer, filename)
# Generate a confirmation log entry.
msg = genLog("NGAMS_INFO_FILE_REGISTERED",
[filename, piRes.getFileId(), piRes.getFileVersion(),
piRes.getFormat()])
time.sleep(0.005)
regTime = time.time() - reg_start
msg = msg + ". Time: %.3fs." % (regTime)
logger.info(msg, extra={'to_syslog': 1})
except Exception, e:
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)])
logger.error(errMsg)
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(errMsg)
fileFailCount += 1
regTime = time.time() - reg_start
# TODO (rtobar, 2016-01): Why don't we raise an exception here?
# Otherwise the command appears as successful on the
# client-side
# Add the file information in the registration report.
if (emailNotif): regDbm.addIncKey(tmpFileObj)
# Update request status time information.
regTimeAccu += regTime
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
fileCount += 1
def archiveFromFile(srvObj,
filename,
noReplication=0,
mimeType=None,
reqPropsObj=None):
"""
Archive a file directly from a file as source.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
filename: Name of file to archive (string).
noReplication: Flag to enable/disable replication (integer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
Returns: Execution result object of DAPI
"""
T = TRACE()
logger.debug("Archiving file: %s", filename)
logger.debug("Mimetype used is %s", mimeType)
if (reqPropsObj):
logger.debug("Request Properties Object given - using this")
reqPropsObjLoc = reqPropsObj
else:
logger.debug("No Request Properties Object given - creating one")
reqPropsObjLoc = ngamsArchiveUtils.ngamsReqProps.ngamsReqProps()
stagingFile = filename
try:
if (mimeType == None):
mimeType = ngamsHighLevelLib.determineMimeType(
srvObj.getCfg(), filename)
archive_start = time.time()
# Prepare dummy ngamsReqProps object (if an object was not given).
if (not reqPropsObj):
reqPropsObjLoc.setMimeType(mimeType)
reqPropsObjLoc.setStagingFilename(filename)
reqPropsObjLoc.setHttpMethod(NGAMS_HTTP_GET)
reqPropsObjLoc.setCmd(NGAMS_ARCHIVE_CMD)
reqPropsObjLoc.setSize(os.path.getsize(filename))
reqPropsObjLoc.setFileUri(NGAMS_HTTP_FILE_URL + filename)
reqPropsObjLoc.setNoReplication(noReplication)
# If no target disk is defined, find one suitable disk.
if (not reqPropsObjLoc.getTargDiskInfo()):
try:
trgDiskInfo = ngamsArchiveUtils.ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
mimeType, 0,
reqSpace=reqPropsObjLoc.getSize())
reqPropsObjLoc.setTargDiskInfo(trgDiskInfo)
# copy the file to the staging area of the target disk
stagingFile = trgDiskInfo.getMountPoint(
) + '/staging/' + os.path.basename(filename)
cpFile(filename, stagingFile)
reqPropsObjLoc.setStagingFilename(stagingFile)
except Exception, e:
errMsg = str(e) + ". Attempting to archive local file: " +\
filename
ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_NO_DISKS,
"NO DISKS AVAILABLE", errMsg)
raise Exception, errMsg
# Set the log cache to 1 during the handling of the file.
plugIn = srvObj.getMimeTypeDic()[mimeType]
logger.debug("Invoking DAPI: %s to handle file: %s", plugIn,
stagingFile)
plugInMethod = loadPlugInEntryPoint(plugIn)
resMain = plugInMethod(srvObj, reqPropsObjLoc)
# Move the file to final destination.
st = time.time()
mvFile(reqPropsObjLoc.getStagingFilename(),
resMain.getCompleteFilename())
iorate = reqPropsObjLoc.getSize() / (time.time() - st)
ngamsArchiveUtils.postFileRecepHandling(srvObj, reqPropsObjLoc,
resMain, trgDiskInfo)
def ngamsMWA_MIT_FilterPlugin(srvObj,
plugInPars,
filename,
fileId,
fileVersion=-1,
reqPropsObj=None):
"""
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: 0 if the file does not match, 1 if it matches the
conditions (integer/0|1).
"""
pars = ""
if ((plugInPars != "") and (plugInPars != None)):
pars = plugInPars
elif (reqPropsObj != None):
if (reqPropsObj.hasHttpPar("plug_in_pars")):
pars = reqPropsObj.getHttpPar("plug_in_pars")
parDic = ngamsPlugInApi.parseRawPlugInPars(pars)
if (not parDic.has_key("remote_host") or not parDic.has_key("remote_port")
or not parDic.has_key("project_id")):
errMsg = "ngamsMWACheckRemoteFilterPlugin: Missing Plug-In Parameter: " +\
"remote_host / remote_port / project_id"
logger.error(errMsg)
return 0
proj_ids = parDic["project_id"]
if (not (proj_ids and len(proj_ids))):
return 0
for proj_id in proj_ids.split(proj_separator):
eor_list.append("'%s'" % proj_id)
fspi = srvObj.getCfg().getFileStagingPlugIn()
if (not fspi):
offline = -1
else:
logger.info("Invoking FSPI.isFileOffline: %s to check file: %s", fspi,
filename)
isFileOffline = loadPlugInEntryPoint(fspi, 'isFileOffline')
offline = isFileOffline(filename)
try:
if (offline == 1 or offline == -1):
# if the file is on Tape or query error, query db instead, otherwise implicit tape staging will block all other threads!!
logger.debug('File %s appears on Tape, connect to MWA DB to check',
filename)
projId = getProjectIdFromMWADB(fileId)
if (not projId or projId == ''):
logger.error('Cannot get project id from MWA DB for file %s',
fileId)
return 0
projectId = "'%s'" % projId # add single quote to be consistent with FITS header keywords
else:
projectId = pyfits.getval(filename, 'PROJID')
except:
err = "Did not find keyword PROJID in FITS file or PROJID illegal"
errMsg = genLog(
"NGAMS_ER_DAPI_BAD_FILE",
[os.path.basename(filename), "ngamsMWA_MIT_FilterPlugIn", err])
return 0
if (projectId in eor_list):
logger.debug('File %s added', fileId)
return 1
else:
return 0
