def getDpIdInfo(filename): """ Generate the File ID (here DP ID) for the file. filename: Name of FITS file (string). Returns: Tuple containing the value of ARCFILE, the DP ID of the file, and the JD date. The two latter deducted from the ARCFILE keyword (tuple). """ try: keyDic = getFitsKeys(filename, ["ARCFILE"]) arcFile = keyDic["ARCFILE"][0] els = arcFile.split(".") dpId = els[0] + "." + els[1] + "." + els[2] date = els[1].split("T")[0] # Make sure that the files are stored according to JD # (one night is 12am -> 12am). isoTime = '.'.join(els[1:]) ts1 = fromiso8601(isoTime) ts2 = tomjd(ts1) - 0.5 dateDirName = toiso8601(frommjd(ts2), fmt=FMT_DATE_ONLY) return [arcFile, dpId, dateDirName] except: err = "Did not find keyword ARCFILE in FITS file or ARCFILE illegal" errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE", [os.path.basename(filename), "ngamsFitsPlugIn", err]) logger.exception(errMsg) raise
def _handleFileList(srvObj, reqPropsObj, httpRef): """ Handle STATUS?file_list... Command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: The File List ID allocated for this request (string). """ T = TRACE() # Should a lower limit for the ingestion date be taken into account. fromIngDate = None if (reqPropsObj.hasHttpPar("from_ingestion_date")): tmpTromIngDate = reqPropsObj.getHttpPar("from_ingestion_date") fromIngDate = fromiso8601(tmpTromIngDate) # Handle the unique flag. If this is specified, only information for unique # File ID/Version pairs are returned. unique = False if (reqPropsObj.hasHttpPar("unique")): unique = int(reqPropsObj.getHttpPar("unique")) # Dump the file information needed. fileListId = genUniqueId() dbmBaseName = STATUS_FILE_LIST_DBM_TAG % fileListId fileInfoDbmName = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(), dbmBaseName) # Dump the file info from the DB. Deal with uniqueness quickly # (instead of using a new file like before) try: fileInfoDbm = ngamsDbm.ngamsDbm(fileInfoDbmName, 0, 1) fileCount = 1 unique_files = set() for f in srvObj.db.files_in_host(srvObj.getHostId(), from_date=fromIngDate): if unique: key = str('%s_%d' % (f[2], f[3])) if key in unique_files: continue else: key = str(fileCount) fileInfoDbm.add(key, f) fileCount += 1 except Exception as e: rmFile(fileInfoDbmName) msg = "Problem generating file list for STATUS Command. " +\ "Parameters: from_ingestion_date=%s. Error: %s" %\ (str(fromIngDate), str(e)) raise Exception(msg) return fileListId
def ngasTarBallPlugIn(srvObj, reqPropsObj): """ Data Archiving Plug-In to handle archiving of tarballs. srvObj: Reference to NG/AMS Server Object (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus). """ stagingFilename = reqPropsObj.getStagingFilename() logger.info("Plug-In handling data for file with URI: %s", os.path.basename(reqPropsObj.getFileUri())) diskInfo = reqPropsObj.getTargDiskInfo() # Check file. checkTarball(stagingFilename) # Get various information about the file being handled. fileId = os.path.basename(reqPropsObj.getFileUri()) timestamp = fileId[fileId.find(".") + 1:] obsDay = tomjd(fromiso8601(timestamp)) - 0.5 dateDirName = toiso8601(obsDay, fmt=FMT_DATE_ONLY) baseFilename = fileId[0:-4] fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, stagingFilename, fileId, baseFilename, [dateDirName], []) # Generate status. logger.debug("Generating status ...") format = "application/x-tar" fileSize = ngamsPlugInApi.getFileSize(stagingFilename) logger.debug("DAPI finished processing of file") return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename, fileId, fileVersion, format, fileSize, fileSize, "NONE", relPath, diskInfo.getSlotId(), fileExists, complFilename)
def handleOnline(srvObj, reqPropsObj=None): """ Initialize the system. This implies loading the NG/AMS Configuration making the internal initialization from this, obtaining information about the disk configuration. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Void. """ logger.debug("Prepare NGAS and NG/AMS for Online State ...") # Re-load Configuration + check disk configuration. srvObj.loadCfg() hostId = srvObj.getHostId() for stream in srvObj.getCfg().getStreamList(): srvObj.getMimeTypeDic()[stream.getMimeType()] = stream.getPlugIn() # Flush/send out possible retained Email Notification Messages. flushMsg = "NOTE: Distribution of retained Email Notification Messages " +\ "forced at Online" ngamsNotification.checkNotifRetBuf(srvObj.getHostId(), srvObj.getCfg(), 1, flushMsg) # Get possible Subscribers from the DB. subscrList = srvObj.getDb().getSubscriberInfo("", hostId, srvObj.getCfg().getPortNo()) num_bl = srvObj.getDb().getSubscrBackLogCount(hostId, srvObj.getCfg().getPortNo()) #debug_chen if (num_bl > 0): logger.debug('Preset the backlog count to %d', num_bl) srvObj.presetSubcrBackLogCount(num_bl) # TODO: unify this "db-to-object" reading (there is something similar elsewhere, # I'm pretty sure, and hide these low-level details from here logger.info("Creating %d subscrption objects from subscription DB info", len(subscrList)) for subscrInfo in subscrList: start_date = fromiso8601(subscrInfo[5], local=True) if subscrInfo[5] else None last_ingested_date = fromiso8601(subscrInfo[8], local=True) if subscrInfo[8] else None tmpSubscrObj = ngamsSubscriber.ngamsSubscriber( subscrInfo[0], subscrInfo[1], subscrInfo[2], subscrInfo[4], start_date, subscrInfo[6], subscrInfo[7], last_ingested_date, subscrInfo[3]) tmpSubscrObj.setConcurrentThreads(subscrInfo[9]) # Take only subscribers for this NG/AMS Server. if ((tmpSubscrObj.getHostId() == hostId) and (tmpSubscrObj.getPortNo() == srvObj.getCfg().getPortNo())): #srvObj.getSubscriberDic()[tmpSubscrObj.getId()] = tmpSubscrObj #if (srvObj.getDataMoverOnlyActive() and len(srvObj.getSubscriberDic()) > 0): # break #only load one subscriber under the data mover mode srvObj.registerSubscriber(tmpSubscrObj) try: # rtobar, Feb 2018 # # We need to do this import here if we want support both python 2 and 3. # This is due to the following circular dependency: # # ngamsArchiveUtils -> ngamsFileUtils -> ngamsSrvUtils -> ngamsArchiveUtils # # A top-level absolute import is impossible, because # "import ngamsServer.ngamsArchiveUtils" interprets ngamsServer as the # sibling module in python 2 (and thus fails). # A top-level relative import fails in python 2 too. # # TODO: in the long run we should avoid the circular dependency, of course from . import ngamsArchiveUtils # Get information about the disks of this system. srvObj.setDiskDic(getDiskInfo(srvObj, reqPropsObj)) ngamsArchiveUtils.resetDiskSpaceWarnings() # Update disk info in DB. ngamsDiskUtils.checkDisks(hostId, srvObj.getDb(), srvObj.getCfg(), srvObj.getDiskDic()) # Write status file on disk. ngamsDiskUtils.dumpDiskInfoAllDisks(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg()) except Exception: errMsg = "Error occurred while bringing the system Online" logger.exception(errMsg) handleOffline(srvObj) raise # Check that there is enough disk space in the various system directories. srvObj.checkDiskSpaceSat() # Check if files are located in the Staging Areas of the Disks. In case # yes, move them to the Back-Log Area to have them treated later. checkStagingAreas(srvObj) # Start threads + inform threads that they are allowed to execute. srvObj.setThreadRunPermission(1) srvObj.startJanitorThread() srvObj.startDataCheckThread() ngamsSubscriptionThread.startSubscriptionThread(srvObj) srvObj.startUserServiceThread() srvObj.startMirControlThread() srvObj.startCacheControlThread() # Change state to Online. srvObj.setState(NGAMS_ONLINE_STATE) # If specified in the configuration file that this server should act as a # Subscriber, then subscribe to the Data Providers specified. We do this # in a thread so that the server can continuously try to subscribe, # even after errors occur. if (srvObj.getCfg().getSubscrEnable()): srvObj.remote_subscription_creation_evt = threading.Event() thrObj = threading.Thread( target=_create_remote_subscriptions, name=NGAMS_SUBSCRIBER_THR, args=(srvObj, srvObj.remote_subscription_creation_evt)) thrObj.setDaemon(0) thrObj.start() logger.info("NG/AMS prepared for Online State")
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle the update subscriber (USUBSCRIBE) Command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ T = TRACE() errMsg = '' err = 0 if (not reqPropsObj.hasHttpPar("subscr_id")): httpRef.send_status("USUBSCRIBE command failed: 'subscr_id' is not specified", status=NGAMS_FAILURE, code=NGAMS_HTTP_SUCCESS) return subscrId = reqPropsObj.getHttpPar("subscr_id") if (not srvObj.getSubscriberDic().has_key(subscrId)): httpRef.send_status("USUBSCRIBE command failed: Cannot find subscriber '%s'" % subscrId, status=NGAMS_FAILURE, code=NGAMS_HTTP_SUCCESS) return if (reqPropsObj.hasHttpPar("suspend")): suspend = int(reqPropsObj.getHttpPar("suspend")) suspend_processed = 0 # could use locks, but the race condition should not really matter here if only one request of suspend is issued at a time (by a system admin?) if (suspend == 1 and srvObj._subscrSuspendDic[subscrId].is_set()): # suspend condition met srvObj._subscrSuspendDic[subscrId].clear() suspend_processed = 1 action = 'SUSPENDED' elif (suspend == 0 and (not srvObj._subscrSuspendDic[subscrId].is_set())): # resume condition met srvObj._subscrSuspendDic[subscrId].set() suspend_processed = 1 action = 'RESUMED' if (suspend_processed): httpRef.send_status("Successfully %s for the subscriber %s" % (action, subscrId)) else: reMsg = "No suspend/resume action is taken for the subscriber %s" % subscrId httpRef.send_status(reMsg, status=NGAMS_FAILURE, code=NGAMS_HTTP_SUCCESS) return subscriber = srvObj.getSubscriberDic()[subscrId] if (reqPropsObj.hasHttpPar("priority")): priority = int(reqPropsObj.getHttpPar("priority")) subscriber.setPriority(priority) if (reqPropsObj.hasHttpPar("url")): url = reqPropsObj.getHttpPar("url") ngamsSubscriber.validate_url(url) subscriber.setUrl(url) if (reqPropsObj.hasHttpPar("start_date")): startDate = reqPropsObj.getHttpPar("start_date").strip() if (startDate): subscriber.setStartDate(fromiso8601(startDate)) lastIngDate = subscriber.getLastFileIngDate() if lastIngDate is not None: # either re-check past files or skip unchecked files subscriber.setLastFileIngDate(None) if (srvObj._subscrScheduledStatus.has_key(subscrId)): #if (startDate < srvObj._subscrScheduledStatus[subscrId] and srvObj._subscrScheduledStatus[subscrId]): # enables trigger re-delivering files that have been previously delivered del srvObj._subscrScheduledStatus[subscrId] if (srvObj._subscrCheckedStatus.has_key(subscrId)): del srvObj._subscrCheckedStatus[subscrId] #if (srvObj._subscrScheduledStatus[subscrId]):# either re-check past files or skip unchecked files #del srvObj._subscrScheduledStatus[subscrId] #srvObj._subscrScheduledStatus[subscrId] = None if (reqPropsObj.hasHttpPar("filter_plug_in")): filterPi = reqPropsObj.getHttpPar("filter_plug_in") subscriber.setFilterPi(filterPi) if (reqPropsObj.hasHttpPar("plug_in_pars")): pipars = reqPropsObj.getHttpPar("plug_in_pars") subscriber.setFilterPiPars(pipars) if (reqPropsObj.hasHttpPar("concurrent_threads")): ccthrds = int(reqPropsObj.getHttpPar("concurrent_threads")) origthrds = int(subscriber.getConcurrentThreads()) if (ccthrds != origthrds): subscriber.setConcurrentThreads(ccthrds) try: changeNumThreads(srvObj, subscrId, origthrds, ccthrds) except Exception, e: msg = " Exception updating subscriber's concurrent threads: %s." % str(e) logger.warning(msg) err += 1 errMsg += msg
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle SUBSCRIBE Command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ T = TRACE() """ if (srvObj.getDataMoverOnlyActive() and len(srvObj.getSubscriberDic()) > 0): srvObj.reply(reqPropsObj, httpRef, NGAMS_HTTP_SUCCESS, NGAMS_FAILURE, "Data Mover NGAS server can have only one subscriber. Use command USUBSCRIBE to update an existing subscriber.") return """ priority = 10 url = "" startDate = time.time() filterPi = "" filterPiPars = "" if (reqPropsObj.hasHttpPar("priority")): priority = reqPropsObj.getHttpPar("priority") if (reqPropsObj.hasHttpPar("url")): url = reqPropsObj.getHttpPar("url") ngamsSubscriber.validate_url(url) else: errMsg = genLog("NGAMS_ER_CMD_SYNTAX", [NGAMS_SUBSCRIBE_CMD, "Missing parameter: url"]) raise Exception, errMsg if (reqPropsObj.hasHttpPar("start_date")): tmpStartDate = reqPropsObj.getHttpPar("start_date").strip() if tmpStartDate: startDate = fromiso8601(tmpStartDate, local=True) if (reqPropsObj.hasHttpPar("filter_plug_in")): filterPi = reqPropsObj.getHttpPar("filter_plug_in") if (reqPropsObj.hasHttpPar("plug_in_pars")): filterPiPars = reqPropsObj.getHttpPar("plug_in_pars") if (reqPropsObj.hasHttpPar("subscr_id")): id = reqPropsObj.getHttpPar("subscr_id") else: id = ngamsLib.getSubscriberId(url) logger.info("Creating subscription for files >= %s", toiso8601(startDate)) subscrObj = ngamsSubscriber.ngamsSubscriber(srvObj.getHostId(), srvObj.getCfg().getPortNo(), priority, url, startDate, filterPi, filterPiPars, subscrId=id) # supports concurrent file transfer, added by [email protected] if (reqPropsObj.hasHttpPar("concurrent_threads")): concurthrds = reqPropsObj.getHttpPar("concurrent_threads") subscrObj.setConcurrentThreads(concurthrds) # If the Start Date given in before the Last Ingestion Date, we # reset the Last Ingestion Date subscrStat = srvObj.getDb().getSubscriberStatus([subscrObj.getId()], subscrObj.getHostId(), subscrObj.getPortNo()) if subscrStat: lastIngDate = subscrStat[0][1] if startDate < lastIngDate: subscrObj.setLastFileIngDate(None) else: subscrObj.setLastFileIngDate(lastIngDate) # Register the Subscriber. addSubscriber(srvObj, subscrObj) # Trigger the Data Susbcription Thread to make it check if there are # files to deliver to the new Subscriber. srvObj.addSubscriptionInfo([], [subscrObj]).triggerSubscriptionThread() return "Handled SUBSCRIBE command"
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle SUBSCRIBE Command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ """ if (srvObj.getDataMoverOnlyActive() and len(srvObj.getSubscriberDic()) > 0): srvObj.reply(reqPropsObj, httpRef, NGAMS_HTTP_SUCCESS, NGAMS_FAILURE, "Data Mover NGAS server can have only one subscriber. Use command USUBSCRIBE to update an existing subscriber.") return """ priority = 10 url = "" startDate = time.time() filterPi = "" filterPiPars = "" if (reqPropsObj.hasHttpPar("priority")): priority = reqPropsObj.getHttpPar("priority") if (reqPropsObj.hasHttpPar("url")): url = reqPropsObj.getHttpPar("url") ngamsSubscriber.validate_url(url) else: errMsg = genLog("NGAMS_ER_CMD_SYNTAX", [NGAMS_SUBSCRIBE_CMD, "Missing parameter: url"]) raise Exception(errMsg) if (reqPropsObj.hasHttpPar("start_date")): tmpStartDate = reqPropsObj.getHttpPar("start_date").strip() if tmpStartDate: startDate = fromiso8601(tmpStartDate, local=True) if (reqPropsObj.hasHttpPar("filter_plug_in")): filterPi = reqPropsObj.getHttpPar("filter_plug_in") if (reqPropsObj.hasHttpPar("plug_in_pars")): filterPiPars = reqPropsObj.getHttpPar("plug_in_pars") if (reqPropsObj.hasHttpPar("subscr_id")): id = reqPropsObj.getHttpPar("subscr_id") else: id = ngamsLib.getSubscriberId(url) logger.info("Creating subscription for files >= %s", toiso8601(startDate)) subscrObj = ngamsSubscriber.ngamsSubscriber(srvObj.getHostId(), srvObj.getCfg().getPortNo(), priority, url, startDate, filterPi, filterPiPars, subscrId=id) # supports concurrent file transfer, added by [email protected] if (reqPropsObj.hasHttpPar("concurrent_threads")): concurthrds = reqPropsObj.getHttpPar("concurrent_threads") subscrObj.setConcurrentThreads(concurthrds) # If the Start Date given in before the Last Ingestion Date, we # reset the Last Ingestion Date # # TODO (rtobar, May 2021): # This bit of logic seems to be in contention with the fact that adding # a subscription with an ID that is already taken would be otherwise a # failure. I'm leaving it here for the time being in order to avoid # breaking anything, but it must probably be removed (or clarification # should be sought otherwise). subscrStat = srvObj.getDb().getSubscriberStatus([subscrObj.getId()], subscrObj.getHostId(), subscrObj.getPortNo()) if subscrStat and subscrStat[0][1]: lastIngDate = fromiso8601(subscrStat[0][1], local=True) if startDate < lastIngDate: subscrObj.setLastFileIngDate('') else: subscrObj.setLastFileIngDate(lastIngDate) # Register the Subscriber. existence_test = addSubscriber(srvObj, subscrObj) if existence_test == 'equal': return 201, NGAMS_SUCCESS, "Identical subscription with ID '%s' existed" % ( id, ) elif existence_test == 'unequal': return 409, NGAMS_FAILURE, "Different subscription with ID '%s' existed" % ( id, ) return "Handled SUBSCRIBE command"
def test_WakeUpDataCheck_1(self): """ Synopsis: Check that a suspended sub-node is woken up when DCC is due and that the DCC is executed as expected after the node has been woken up. Description: Before suspending itself, a sub-node should request to be woken up when the time for executing the next DCC is due. The purpose of the test is to verify that a sub-node suspending itself requests to be woken up at the specified point in time, and that the contacted Master Node wakes up a suspended sub-node as expected. Expected Result: The contacted Master Node should identify that the sub-node is suspended and that it has requested to be woken up by the Master Node at a given point in time. The Master Node should wake up the node at the requested point in time and the sub-node should carry out the DCC. Test Steps: - Prepare simulated cluster with a sub-node suspending itself after a short while and having DCC enabled with a high frequency. - Archive two files onto the sub-node. - Wait till the sub-node has suspended itself. - Wait till the sub-node has been woken up. - Check that entries in the log file on the Master Node indicate that it has woken up the suspended sub-node. - Check that entries in the log file on the sub-node indicate that it has been woken up. - Wait until the DCC has been executed and check that the result is as expected (the DCC summary log appears in the log file on the sub-node). Remarks: ... """ masterNode, susp_nodes = get_nodes() subNode1 = susp_nodes[0] # Always delete sub-node log file (to run test with a fresh log file). rmFile(subNode1Log) # Enable DCC in 8001 + define a minimum check cycle of 15s. datacheck_period = 15 cfgParDic = { 8001: (("NgamsCfg.DataCheckThread[1].Active", "1"), ("NgamsCfg.DataCheckThread[1].MinCycle", "00T00:00:%d" % datacheck_period)) } dbConObj = prepSimCluster(self, cfgParDic=cfgParDic)[masterNode][1] # Archive two files into subnode 8001 and wait for it to go to suspended state self.archive(8001, "src/TinyTestFile.fits") self.archive(8001, "src/SmallFile.fits") suspension_time = self.waitTillSuspended(dbConObj, subNode1, 30, susp_nodes) # Now we have to wait until the sub-node is woken up for DCC. self.waitTillWokenUp(dbConObj, subNode1, 60, susp_nodes) # Check log output in Master Log File. tagFormat = "Waking up server %s" testTags = [tagFormat % (subNode1, )] self.checkTags(loadFile(masterNodeLog), testTags, showBuf=0) # Check log output in Sub-Node Log File. self.status(8001) # Flush log cache. time.sleep(2) # and wait a bit testTags = ["NGAS Node: %s woken up after" % subNode1] self.checkTags(loadFile(subNode1Log), testTags, showBuf=0) # Wait until the DCC has been executed. startTime = time.time() datacheck_executed = False dccStatLog = None while (time.time() - startTime) < (datacheck_period * 1.5): datacheck_time = None for line in reversed(loadFile(subNode1Log).splitlines()): if "NGAMS_INFO_DATA_CHK_STAT" in line: dccStatLog = line datacheck_time = dccStatLog.split(" ")[0] break if datacheck_time and fromiso8601( datacheck_time) < suspension_time: datacheck_executed = True break time.sleep(2) if not datacheck_executed: self.fail("Data Consistency Checking not executed AFTER host " +\ "wake-up as expected")