def getDpIdInfo(filename):
"""
Generate the File ID (here DP ID) for the file.
filename: Name of FITS file (string).
Returns: Tuple containing the value of ARCFILE, the DP ID
of the file, and the JD date. The two latter deducted from
the ARCFILE keyword (tuple).
"""
try:
keyDic = getFitsKeys(filename, ["ARCFILE"])
arcFile = keyDic["ARCFILE"][0]
els = arcFile.split(".")
dpId = els[0] + "." + els[1] + "." + els[2]
date = els[1].split("T")[0]
# Make sure that the files are stored according to JD
# (one night is 12am -> 12am).
isoTime = '.'.join(els[1:])
ts1 = fromiso8601(isoTime)
ts2 = tomjd(ts1) - 0.5
dateDirName = toiso8601(frommjd(ts2), fmt=FMT_DATE_ONLY)
return [arcFile, dpId, dateDirName]
except:
err = "Did not find keyword ARCFILE in FITS file or ARCFILE illegal"
errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE",
[os.path.basename(filename), "ngamsFitsPlugIn", err])
logger.exception(errMsg)
raise
def _handleFileList(srvObj, reqPropsObj, httpRef):
"""
Handle STATUS?file_list... Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: The File List ID allocated for this request (string).
"""
T = TRACE()
# Should a lower limit for the ingestion date be taken into account.
fromIngDate = None
if (reqPropsObj.hasHttpPar("from_ingestion_date")):
tmpTromIngDate = reqPropsObj.getHttpPar("from_ingestion_date")
fromIngDate = fromiso8601(tmpTromIngDate)
# Handle the unique flag. If this is specified, only information for unique
# File ID/Version pairs are returned.
unique = False
if (reqPropsObj.hasHttpPar("unique")):
unique = int(reqPropsObj.getHttpPar("unique"))
# Dump the file information needed.
fileListId = genUniqueId()
dbmBaseName = STATUS_FILE_LIST_DBM_TAG % fileListId
fileInfoDbmName = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(),
dbmBaseName)
# Dump the file info from the DB. Deal with uniqueness quickly
# (instead of using a new file like before)
try:
fileInfoDbm = ngamsDbm.ngamsDbm(fileInfoDbmName, 0, 1)
fileCount = 1
unique_files = set()
for f in srvObj.db.files_in_host(srvObj.getHostId(),
from_date=fromIngDate):
if unique:
key = str('%s_%d' % (f[2], f[3]))
if key in unique_files:
continue
else:
key = str(fileCount)
fileInfoDbm.add(key, f)
fileCount += 1
except Exception as e:
rmFile(fileInfoDbmName)
msg = "Problem generating file list for STATUS Command. " +\
"Parameters: from_ingestion_date=%s. Error: %s" %\
(str(fromIngDate), str(e))
raise Exception(msg)
return fileListId
def ngasTarBallPlugIn(srvObj, reqPropsObj):
"""
Data Archiving Plug-In to handle archiving of tarballs.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Standard NG/AMS Data Archiving Plug-In Status
as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus).
"""
stagingFilename = reqPropsObj.getStagingFilename()
logger.info("Plug-In handling data for file with URI: %s",
os.path.basename(reqPropsObj.getFileUri()))
diskInfo = reqPropsObj.getTargDiskInfo()
# Check file.
checkTarball(stagingFilename)
# Get various information about the file being handled.
fileId = os.path.basename(reqPropsObj.getFileUri())
timestamp = fileId[fileId.find(".") + 1:]
obsDay = tomjd(fromiso8601(timestamp)) - 0.5
dateDirName = toiso8601(obsDay, fmt=FMT_DATE_ONLY)
baseFilename = fileId[0:-4]
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(),
reqPropsObj, diskInfo,
stagingFilename, fileId,
baseFilename, [dateDirName], [])
# Generate status.
logger.debug("Generating status ...")
format = "application/x-tar"
fileSize = ngamsPlugInApi.getFileSize(stagingFilename)
logger.debug("DAPI finished processing of file")
return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename,
fileId, fileVersion, format,
fileSize, fileSize,
"NONE", relPath,
diskInfo.getSlotId(), fileExists,
complFilename)
def handleOnline(srvObj, reqPropsObj=None):
"""
Initialize the system. This implies loading the NG/AMS
Configuration making the internal initialization from this,
obtaining information about the disk configuration.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Void.
"""
logger.debug("Prepare NGAS and NG/AMS for Online State ...")
# Re-load Configuration + check disk configuration.
srvObj.loadCfg()
hostId = srvObj.getHostId()
for stream in srvObj.getCfg().getStreamList():
srvObj.getMimeTypeDic()[stream.getMimeType()] = stream.getPlugIn()
# Flush/send out possible retained Email Notification Messages.
flushMsg = "NOTE: Distribution of retained Email Notification Messages " +\
"forced at Online"
ngamsNotification.checkNotifRetBuf(srvObj.getHostId(), srvObj.getCfg(), 1,
flushMsg)
# Get possible Subscribers from the DB.
subscrList = srvObj.getDb().getSubscriberInfo("", hostId,
srvObj.getCfg().getPortNo())
num_bl = srvObj.getDb().getSubscrBackLogCount(hostId,
srvObj.getCfg().getPortNo())
#debug_chen
if (num_bl > 0):
logger.debug('Preset the backlog count to %d', num_bl)
srvObj.presetSubcrBackLogCount(num_bl)
# TODO: unify this "db-to-object" reading (there is something similar elsewhere,
# I'm pretty sure, and hide these low-level details from here
logger.info("Creating %d subscrption objects from subscription DB info",
len(subscrList))
for subscrInfo in subscrList:
start_date = fromiso8601(subscrInfo[5],
local=True) if subscrInfo[5] else None
last_ingested_date = fromiso8601(subscrInfo[8],
local=True) if subscrInfo[8] else None
tmpSubscrObj = ngamsSubscriber.ngamsSubscriber(
subscrInfo[0], subscrInfo[1], subscrInfo[2], subscrInfo[4],
start_date, subscrInfo[6], subscrInfo[7], last_ingested_date,
subscrInfo[3])
tmpSubscrObj.setConcurrentThreads(subscrInfo[9])
# Take only subscribers for this NG/AMS Server.
if ((tmpSubscrObj.getHostId() == hostId)
and (tmpSubscrObj.getPortNo() == srvObj.getCfg().getPortNo())):
#srvObj.getSubscriberDic()[tmpSubscrObj.getId()] = tmpSubscrObj
#if (srvObj.getDataMoverOnlyActive() and len(srvObj.getSubscriberDic()) > 0):
# break #only load one subscriber under the data mover mode
srvObj.registerSubscriber(tmpSubscrObj)
try:
# rtobar, Feb 2018
#
# We need to do this import here if we want support both python 2 and 3.
# This is due to the following circular dependency:
#
# ngamsArchiveUtils -> ngamsFileUtils -> ngamsSrvUtils -> ngamsArchiveUtils
#
# A top-level absolute import is impossible, because
# "import ngamsServer.ngamsArchiveUtils" interprets ngamsServer as the
# sibling module in python 2 (and thus fails).
# A top-level relative import fails in python 2 too.
#
# TODO: in the long run we should avoid the circular dependency, of course
from . import ngamsArchiveUtils
# Get information about the disks of this system.
srvObj.setDiskDic(getDiskInfo(srvObj, reqPropsObj))
ngamsArchiveUtils.resetDiskSpaceWarnings()
# Update disk info in DB.
ngamsDiskUtils.checkDisks(hostId, srvObj.getDb(), srvObj.getCfg(),
srvObj.getDiskDic())
# Write status file on disk.
ngamsDiskUtils.dumpDiskInfoAllDisks(srvObj.getHostId(), srvObj.getDb(),
srvObj.getCfg())
except Exception:
errMsg = "Error occurred while bringing the system Online"
logger.exception(errMsg)
handleOffline(srvObj)
raise
# Check that there is enough disk space in the various system directories.
srvObj.checkDiskSpaceSat()
# Check if files are located in the Staging Areas of the Disks. In case
# yes, move them to the Back-Log Area to have them treated later.
checkStagingAreas(srvObj)
# Start threads + inform threads that they are allowed to execute.
srvObj.setThreadRunPermission(1)
srvObj.startJanitorThread()
srvObj.startDataCheckThread()
ngamsSubscriptionThread.startSubscriptionThread(srvObj)
srvObj.startUserServiceThread()
srvObj.startMirControlThread()
srvObj.startCacheControlThread()
# Change state to Online.
srvObj.setState(NGAMS_ONLINE_STATE)
# If specified in the configuration file that this server should act as a
# Subscriber, then subscribe to the Data Providers specified. We do this
# in a thread so that the server can continuously try to subscribe,
# even after errors occur.
if (srvObj.getCfg().getSubscrEnable()):
srvObj.remote_subscription_creation_evt = threading.Event()
thrObj = threading.Thread(
target=_create_remote_subscriptions,
name=NGAMS_SUBSCRIBER_THR,
args=(srvObj, srvObj.remote_subscription_creation_evt))
thrObj.setDaemon(0)
thrObj.start()
logger.info("NG/AMS prepared for Online State")
def handleCmd(srvObj,
reqPropsObj,
httpRef):
"""
Handle the update subscriber (USUBSCRIBE) Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
errMsg = ''
err = 0
if (not reqPropsObj.hasHttpPar("subscr_id")):
httpRef.send_status("USUBSCRIBE command failed: 'subscr_id' is not specified",
status=NGAMS_FAILURE, code=NGAMS_HTTP_SUCCESS)
return
subscrId = reqPropsObj.getHttpPar("subscr_id")
if (not srvObj.getSubscriberDic().has_key(subscrId)):
httpRef.send_status("USUBSCRIBE command failed: Cannot find subscriber '%s'" % subscrId,
status=NGAMS_FAILURE, code=NGAMS_HTTP_SUCCESS)
return
if (reqPropsObj.hasHttpPar("suspend")):
suspend = int(reqPropsObj.getHttpPar("suspend"))
suspend_processed = 0
# could use locks, but the race condition should not really matter here if only one request of suspend is issued at a time (by a system admin?)
if (suspend == 1 and srvObj._subscrSuspendDic[subscrId].is_set()): # suspend condition met
srvObj._subscrSuspendDic[subscrId].clear()
suspend_processed = 1
action = 'SUSPENDED'
elif (suspend == 0 and (not srvObj._subscrSuspendDic[subscrId].is_set())): # resume condition met
srvObj._subscrSuspendDic[subscrId].set()
suspend_processed = 1
action = 'RESUMED'
if (suspend_processed):
httpRef.send_status("Successfully %s for the subscriber %s" % (action, subscrId))
else:
reMsg = "No suspend/resume action is taken for the subscriber %s" % subscrId
httpRef.send_status(reMsg, status=NGAMS_FAILURE, code=NGAMS_HTTP_SUCCESS)
return
subscriber = srvObj.getSubscriberDic()[subscrId]
if (reqPropsObj.hasHttpPar("priority")):
priority = int(reqPropsObj.getHttpPar("priority"))
subscriber.setPriority(priority)
if (reqPropsObj.hasHttpPar("url")):
url = reqPropsObj.getHttpPar("url")
ngamsSubscriber.validate_url(url)
subscriber.setUrl(url)
if (reqPropsObj.hasHttpPar("start_date")):
startDate = reqPropsObj.getHttpPar("start_date").strip()
if (startDate):
subscriber.setStartDate(fromiso8601(startDate))
lastIngDate = subscriber.getLastFileIngDate()
if lastIngDate is not None: # either re-check past files or skip unchecked files
subscriber.setLastFileIngDate(None)
if (srvObj._subscrScheduledStatus.has_key(subscrId)):
#if (startDate < srvObj._subscrScheduledStatus[subscrId] and srvObj._subscrScheduledStatus[subscrId]): # enables trigger re-delivering files that have been previously delivered
del srvObj._subscrScheduledStatus[subscrId]
if (srvObj._subscrCheckedStatus.has_key(subscrId)):
del srvObj._subscrCheckedStatus[subscrId]
#if (srvObj._subscrScheduledStatus[subscrId]):# either re-check past files or skip unchecked files
#del srvObj._subscrScheduledStatus[subscrId]
#srvObj._subscrScheduledStatus[subscrId] = None
if (reqPropsObj.hasHttpPar("filter_plug_in")):
filterPi = reqPropsObj.getHttpPar("filter_plug_in")
subscriber.setFilterPi(filterPi)
if (reqPropsObj.hasHttpPar("plug_in_pars")):
pipars = reqPropsObj.getHttpPar("plug_in_pars")
subscriber.setFilterPiPars(pipars)
if (reqPropsObj.hasHttpPar("concurrent_threads")):
ccthrds = int(reqPropsObj.getHttpPar("concurrent_threads"))
origthrds = int(subscriber.getConcurrentThreads())
if (ccthrds != origthrds):
subscriber.setConcurrentThreads(ccthrds)
try:
changeNumThreads(srvObj, subscrId, origthrds, ccthrds)
except Exception, e:
msg = " Exception updating subscriber's concurrent threads: %s." % str(e)
logger.warning(msg)
err += 1
errMsg += msg
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle SUBSCRIBE Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
"""
if (srvObj.getDataMoverOnlyActive() and len(srvObj.getSubscriberDic()) > 0):
srvObj.reply(reqPropsObj, httpRef, NGAMS_HTTP_SUCCESS, NGAMS_FAILURE,
"Data Mover NGAS server can have only one subscriber. Use command USUBSCRIBE to update an existing subscriber.")
return
"""
priority = 10
url = ""
startDate = time.time()
filterPi = ""
filterPiPars = ""
if (reqPropsObj.hasHttpPar("priority")):
priority = reqPropsObj.getHttpPar("priority")
if (reqPropsObj.hasHttpPar("url")):
url = reqPropsObj.getHttpPar("url")
ngamsSubscriber.validate_url(url)
else:
errMsg = genLog("NGAMS_ER_CMD_SYNTAX",
[NGAMS_SUBSCRIBE_CMD, "Missing parameter: url"])
raise Exception, errMsg
if (reqPropsObj.hasHttpPar("start_date")):
tmpStartDate = reqPropsObj.getHttpPar("start_date").strip()
if tmpStartDate:
startDate = fromiso8601(tmpStartDate, local=True)
if (reqPropsObj.hasHttpPar("filter_plug_in")):
filterPi = reqPropsObj.getHttpPar("filter_plug_in")
if (reqPropsObj.hasHttpPar("plug_in_pars")):
filterPiPars = reqPropsObj.getHttpPar("plug_in_pars")
if (reqPropsObj.hasHttpPar("subscr_id")):
id = reqPropsObj.getHttpPar("subscr_id")
else:
id = ngamsLib.getSubscriberId(url)
logger.info("Creating subscription for files >= %s", toiso8601(startDate))
subscrObj = ngamsSubscriber.ngamsSubscriber(srvObj.getHostId(),
srvObj.getCfg().getPortNo(),
priority,
url,
startDate,
filterPi,
filterPiPars,
subscrId=id)
# supports concurrent file transfer, added by [email protected]
if (reqPropsObj.hasHttpPar("concurrent_threads")):
concurthrds = reqPropsObj.getHttpPar("concurrent_threads")
subscrObj.setConcurrentThreads(concurthrds)
# If the Start Date given in before the Last Ingestion Date, we
# reset the Last Ingestion Date
subscrStat = srvObj.getDb().getSubscriberStatus([subscrObj.getId()],
subscrObj.getHostId(),
subscrObj.getPortNo())
if subscrStat:
lastIngDate = subscrStat[0][1]
if startDate < lastIngDate:
subscrObj.setLastFileIngDate(None)
else:
subscrObj.setLastFileIngDate(lastIngDate)
# Register the Subscriber.
addSubscriber(srvObj, subscrObj)
# Trigger the Data Susbcription Thread to make it check if there are
# files to deliver to the new Subscriber.
srvObj.addSubscriptionInfo([], [subscrObj]).triggerSubscriptionThread()
return "Handled SUBSCRIBE command"
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle SUBSCRIBE Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
"""
if (srvObj.getDataMoverOnlyActive() and len(srvObj.getSubscriberDic()) > 0):
srvObj.reply(reqPropsObj, httpRef, NGAMS_HTTP_SUCCESS, NGAMS_FAILURE,
"Data Mover NGAS server can have only one subscriber. Use command USUBSCRIBE to update an existing subscriber.")
return
"""
priority = 10
url = ""
startDate = time.time()
filterPi = ""
filterPiPars = ""
if (reqPropsObj.hasHttpPar("priority")):
priority = reqPropsObj.getHttpPar("priority")
if (reqPropsObj.hasHttpPar("url")):
url = reqPropsObj.getHttpPar("url")
ngamsSubscriber.validate_url(url)
else:
errMsg = genLog("NGAMS_ER_CMD_SYNTAX",
[NGAMS_SUBSCRIBE_CMD, "Missing parameter: url"])
raise Exception(errMsg)
if (reqPropsObj.hasHttpPar("start_date")):
tmpStartDate = reqPropsObj.getHttpPar("start_date").strip()
if tmpStartDate:
startDate = fromiso8601(tmpStartDate, local=True)
if (reqPropsObj.hasHttpPar("filter_plug_in")):
filterPi = reqPropsObj.getHttpPar("filter_plug_in")
if (reqPropsObj.hasHttpPar("plug_in_pars")):
filterPiPars = reqPropsObj.getHttpPar("plug_in_pars")
if (reqPropsObj.hasHttpPar("subscr_id")):
id = reqPropsObj.getHttpPar("subscr_id")
else:
id = ngamsLib.getSubscriberId(url)
logger.info("Creating subscription for files >= %s", toiso8601(startDate))
subscrObj = ngamsSubscriber.ngamsSubscriber(srvObj.getHostId(),
srvObj.getCfg().getPortNo(),
priority,
url,
startDate,
filterPi,
filterPiPars,
subscrId=id)
# supports concurrent file transfer, added by [email protected]
if (reqPropsObj.hasHttpPar("concurrent_threads")):
concurthrds = reqPropsObj.getHttpPar("concurrent_threads")
subscrObj.setConcurrentThreads(concurthrds)
# If the Start Date given in before the Last Ingestion Date, we
# reset the Last Ingestion Date
#
# TODO (rtobar, May 2021):
# This bit of logic seems to be in contention with the fact that adding
# a subscription with an ID that is already taken would be otherwise a
# failure. I'm leaving it here for the time being in order to avoid
# breaking anything, but it must probably be removed (or clarification
# should be sought otherwise).
subscrStat = srvObj.getDb().getSubscriberStatus([subscrObj.getId()],
subscrObj.getHostId(),
subscrObj.getPortNo())
if subscrStat and subscrStat[0][1]:
lastIngDate = fromiso8601(subscrStat[0][1], local=True)
if startDate < lastIngDate:
subscrObj.setLastFileIngDate('')
else:
subscrObj.setLastFileIngDate(lastIngDate)
# Register the Subscriber.
existence_test = addSubscriber(srvObj, subscrObj)
if existence_test == 'equal':
return 201, NGAMS_SUCCESS, "Identical subscription with ID '%s' existed" % (
id, )
elif existence_test == 'unequal':
return 409, NGAMS_FAILURE, "Different subscription with ID '%s' existed" % (
id, )
return "Handled SUBSCRIBE command"
def test_WakeUpDataCheck_1(self):
"""
Synopsis:
Check that a suspended sub-node is woken up when DCC is due and that
the DCC is executed as expected after the node has been woken up.
Description:
Before suspending itself, a sub-node should request to be woken up
when the time for executing the next DCC is due.
The purpose of the test is to verify that a sub-node suspending itself
requests to be woken up at the specified point in time, and that the
contacted Master Node wakes up a suspended sub-node as expected.
Expected Result:
The contacted Master Node should identify that the sub-node is
suspended and that it has requested to be woken up by the Master Node
at a given point in time. The Master Node should wake up the node at
the requested point in time and the sub-node should carry out the DCC.
Test Steps:
- Prepare simulated cluster with a sub-node suspending itself after a
short while and having DCC enabled with a high frequency.
- Archive two files onto the sub-node.
- Wait till the sub-node has suspended itself.
- Wait till the sub-node has been woken up.
- Check that entries in the log file on the Master Node indicate
that it has woken up the suspended sub-node.
- Check that entries in the log file on the sub-node indicate that it
has been woken up.
- Wait until the DCC has been executed and check that the result is
as expected (the DCC summary log appears in the log file on the
sub-node).
Remarks:
...
"""
masterNode, susp_nodes = get_nodes()
subNode1 = susp_nodes[0]
# Always delete sub-node log file (to run test with a fresh log file).
rmFile(subNode1Log)
# Enable DCC in 8001 + define a minimum check cycle of 15s.
datacheck_period = 15
cfgParDic = {
8001: (("NgamsCfg.DataCheckThread[1].Active", "1"),
("NgamsCfg.DataCheckThread[1].MinCycle",
"00T00:00:%d" % datacheck_period))
}
dbConObj = prepSimCluster(self, cfgParDic=cfgParDic)[masterNode][1]
# Archive two files into subnode 8001 and wait for it to go to suspended state
self.archive(8001, "src/TinyTestFile.fits")
self.archive(8001, "src/SmallFile.fits")
suspension_time = self.waitTillSuspended(dbConObj, subNode1, 30,
susp_nodes)
# Now we have to wait until the sub-node is woken up for DCC.
self.waitTillWokenUp(dbConObj, subNode1, 60, susp_nodes)
# Check log output in Master Log File.
tagFormat = "Waking up server %s"
testTags = [tagFormat % (subNode1, )]
self.checkTags(loadFile(masterNodeLog), testTags, showBuf=0)
# Check log output in Sub-Node Log File.
self.status(8001) # Flush log cache.
time.sleep(2) # and wait a bit
testTags = ["NGAS Node: %s woken up after" % subNode1]
self.checkTags(loadFile(subNode1Log), testTags, showBuf=0)
# Wait until the DCC has been executed.
startTime = time.time()
datacheck_executed = False
dccStatLog = None
while (time.time() - startTime) < (datacheck_period * 1.5):
datacheck_time = None
for line in reversed(loadFile(subNode1Log).splitlines()):
if "NGAMS_INFO_DATA_CHK_STAT" in line:
dccStatLog = line
datacheck_time = dccStatLog.split(" ")[0]
break
if datacheck_time and fromiso8601(
datacheck_time) < suspension_time:
datacheck_executed = True
break
time.sleep(2)
if not datacheck_executed:
self.fail("Data Consistency Checking not executed AFTER host " +\
"wake-up as expected")
