コード例 #1
0
ファイル: ngamsRemUtils.py プロジェクト: ulange-eso-org/ngas
def _notify(srvObj, reqPropsObj, statRep, mimeType, cmd):
    """
    Prepare and send an Email Notification Message.

    srvObj:         Reference to NG/AMS server class object (ngamsServer).

    reqPropsObj:    Request Property object to keep track of actions done
                    during the request handling (ngamsReqProps).

    statRep:        Status report filename (string).

    mimeType:       Mime-type of data to send (string/mime-type).

    Returns:        Void.
    """
    emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",")
    attachmentName = cmd + "-StatusReport"
    attachmentName += "-" + reqPropsObj.getHttpPar("disk_id")
    ngamsNotification.notify(srvObj.host_id,
                             srvObj.cfg,
                             NGAMS_NOTIF_INFO,
                             cmd + " STATUS REPORT",
                             statRep,
                             recList=emailAdrList,
                             force=1,
                             contentType=mimeType,
                             attachmentName=attachmentName)
コード例 #2
0
def run(srvObj, stopEvt):

    hostId = srvObj.getHostId()
    cfg = srvObj.getCfg()

    now = time.time()
    suspend = (cfg.getIdleSuspension() and
               not srvObj.getHandlingCmd() and
               not srvObj.getDb().getSrvDataChecking(hostId) and
               now - srvObj.getLastReqEndTime() >= cfg.getIdleSuspensionTime())

    if not suspend:
        return

    # Conditions are met for suspending this NGAS host.
    logger.info("NG/AMS Server %s suspending itself", hostId)

    # If Data Checking is on, we request a wake-up call.
    if cfg.getDataCheckActive():
        wakeUpSrv = cfg.getWakeUpServerHost()
        nextDataCheck = srvObj.getNextDataCheckTime()
        srvObj.reqWakeUpCall(wakeUpSrv, nextDataCheck)

    # Now, suspend this host.
    srvObj.getDb().markHostSuspended(hostId)
    suspPi = cfg.getSuspensionPlugIn()
    logger.debug("Invoking Suspension Plug-In: %s to " +\
         "suspend NG/AMS Server: %s", suspPi, hostId)
    try:
        plugInMethod = loadPlugInEntryPoint(suspPi)
        plugInMethod(srvObj)
    except Exception, e:
        errMsg = "Error suspending server with Suspension Plug-In %s"
        logger.exception(errMsg, suspPi)
        ngamsNotification.notify(hostId, cfg, NGAMS_NOTIF_ERROR,
                                 "ERROR INVOKING SUSPENSION "+\
                                 "PLUG-IN", errMsg % (suspPi,) + ": %s" % (str(e),))
        raise
コード例 #3
0
            fo.write(149 * "-")
            fo.write("\n\n==END\n")
            fo.close()
            mimeType = NGAMS_TEXT_MT

        # Send out the status report.
        emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",")
        attachmentName = "CloneStatusReport"
        if (reqPropsObj.hasHttpPar("disk_id")):
            attachmentName += "-" + reqPropsObj.getHttpPar("disk_id")
        if (reqPropsObj.hasHttpPar("file_id")):
            attachmentName += "-" + reqPropsObj.getHttpPar("file_id")
        if (reqPropsObj.hasHttpPar("file_version")):
            attachmentName += "-" + reqPropsObj.getHttpPar("file_version")
        ngamsNotification.notify(srvObj.getHostId(), srvObj.getCfg(),
                                 NGAMS_NOTIF_INFO, "CLONE STATUS REPORT",
                                 statRep, emailAdrList, 1, mimeType,
                                 attachmentName, 1)
        del cloneStatusDbm
        rmFile(cloneStatusDbmName + "*")
        rmFile(statRep)

    if (cloneListDbm): del cloneListDbm
    rmFile(cloneListDbmName + "*")
    logger.info(
        "_cloneExec(). Total time: %.3fs. Average time per file: %.3fs.",
        timeAccu, (timeAccu / totFiles))


def _cloneExplicit(srvObj, reqPropsObj, diskId, fileId, fileVersion,
                   targetDiskId):
    """
コード例 #4
0
ファイル: ngamsDbSnapshotUtils.py プロジェクト: 33bunny/ngas
def checkUpdateDbSnapShots(srvObj, stopEvt):
    """
    Check if a DB Snapshot exists for the DB connected. If not, this is
    created according to the contents of the NGAS DB (if possible). During
    this creation it is checked if the file are physically stored on the
    disk.

    srvObj:        Reference to NG/AMS server class object (ngamsServer).

    Returns:       Void.
    """
    snapshotDbm = None
    tmpSnapshotDbm = None

    if (not srvObj.getCfg().getDbSnapshot()):
        logger.debug("NOTE: DB Snapshot Feature is switched off")
        return

    logger.debug("Generate list of disks to check ...")
    tmpDiskIdMtPtList = srvObj.getDb().getDiskIdsMtPtsMountedDisks(
        srvObj.getHostId())
    diskIdMtPtList = []
    for diskId, mtPt in tmpDiskIdMtPtList:
        diskIdMtPtList.append([mtPt, diskId])
    diskIdMtPtList.sort()
    logger.debug("Generated list of disks to check: %s", str(diskIdMtPtList))

    # Generate temporary snapshot filename.
    ngasId = srvObj.getHostId()
    tmpDir = ngamsHighLevelLib.getTmpDir(srvObj.getCfg())

    # Temporary DBM with file info from the DB.
    tmpSnapshotDbmName = os.path.normpath(tmpDir + "/" + ngasId + "_" +\
                                          NGAMS_DB_NGAS_FILES)

    # Temporary DBM to contain information about 'lost files', i.e. files,
    # which are registered in the DB and found in the DB Snapshot, but
    # which are not found on the disk.
    logger.debug("Create DBM to hold information about lost files ...")
    lostFileRefsDbmName = os.path.normpath(tmpDir + "/" + ngasId +\
                                           "_LOST_FILES")
    rmFile(lostFileRefsDbmName + "*")
    lostFileRefsDbm = ngamsDbm.ngamsDbm(lostFileRefsDbmName, writePerm=1)

    # Carry out the check.
    for mtPt, diskId in diskIdMtPtList:

        checkStopJanitorThread(stopEvt)

        logger.debug("Check/create/update DB Snapshot for disk with " +\
             "mount point: %s", mtPt)

        try:
            snapshotDbm = _openDbSnapshot(srvObj.getCfg(), mtPt)
            if (snapshotDbm == None):
                continue

            # The scheme for synchronizing the Snapshot and the DB is:
            #
            # - Loop over file entries in the Snapshot:
            #  - If in DB:
            #    - If file on disk     -> OK, do nothing.
            #    - If file not on disk -> Accumulate + issue collective warning.
            #
            #  - If entry not in DB:
            #    - If file on disk     -> Add entry in DB.
            #    - If file not on disk -> Remove entry from Snapshot.
            #
            # - Loop over entries for that disk in the DB:
            #  - If entry in Snapshot  -> OK, do nothing.
            #  - If entry not in Snapshot:
            #    - If file on disk     -> Add entry in Snapshot.
            #    - If file not on disk -> Remove entry from DB.

            # Create a temporary DB Snapshot with the files from the DB.
            try:
                rmFile(tmpSnapshotDbmName + "*")
                tmpSnapshotDbm = bsddb.hashopen(tmpSnapshotDbmName, "c")

                for fileInfo in srvObj.db.getFileInfoList(diskId, ignore=None):
                    fileKey = _genFileKey(fileInfo)
                    encFileInfoDic = _encFileInfo(srvObj.getDb(),
                                                  tmpSnapshotDbm, fileInfo)
                    _addInDbm(tmpSnapshotDbm, fileKey, encFileInfoDic)
                    checkStopJanitorThread(stopEvt)
                tmpSnapshotDbm.sync()
            except:
                rmFile(tmpSnapshotDbmName)
                raise

            #####################################################################
            # Loop over the possible entries in the DB Snapshot and compare
            # these against the DB.
            #####################################################################
            logger.debug("Loop over file entries in the DB Snapshot - %s ...",
                         diskId)
            count = 0
            try:
                key, pickleValue = snapshotDbm.first()
            except Exception as e:
                msg = "Exception raised accessing DB Snapshot for disk: %s. " +\
                      "Error: %s"
                logger.debug(msg, diskId, str(e))
                key = None
                snapshotDbm.dbc = None

            # Create a DBM which is used to keep the list of files to remove
            # from the DB Snapshot.
            snapshotDelDbmName = ngamsHighLevelLib.\
                                 genTmpFilename(srvObj.getCfg(),
                                                NGAMS_DB_NGAS_FILES)
            snapshotDelDbm = ngamsDbm.ngamsDbm(snapshotDelDbmName,
                                               cleanUpOnDestr=1,
                                               writePerm=1)

            #################################################################################################
            #jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath
            #          when iterating at the end of the table that are prone to corrupt the hash table object
            #while (key):
            for key, pickleValue in snapshotDbm.iteritems():
                #################################################################################################
                value = cPickle.loads(pickleValue)

                # Check if an administrative element, if yes add it if necessary.
                if b"___" in key:
                    if (not tmpSnapshotDbm.has_key(key)):
                        tmpSnapshotDbm[key] = pickleValue
                else:
                    tmpFileObj = _encFileInfo2Obj(srvObj.getDb(), snapshotDbm,
                                                  value)
                    if (tmpFileObj is None):
                        continue
                    complFilename = os.path.normpath(mtPt + "/" +\
                                                     tmpFileObj.getFilename())

                    # Is the file in the DB?
                    if (tmpSnapshotDbm.has_key(key)):
                        # Is the file on the disk?
                        if (not os.path.exists(complFilename)):
                            fileVer = tmpFileObj.getFileVersion()
                            tmpFileObj.setTag(complFilename)
                            fileKey = ngamsLib.genFileKey(
                                tmpFileObj.getDiskId(), tmpFileObj.getFileId(),
                                fileVer)
                            lostFileRefsDbm.add(fileKey, tmpFileObj)
                            lostFileRefsDbm.sync()
                    elif (not tmpSnapshotDbm.has_key(key)):
                        tmpFileObj = _encFileInfo2Obj(srvObj.getDb(),
                                                      snapshotDbm, value)
                        if (tmpFileObj is None):
                            continue

                        # Is the file on the disk?
                        if (os.path.exists(complFilename)):
                            # Add this entry in the NGAS DB.
                            tmpFileObj.write(srvObj.getHostId(),
                                             srvObj.getDb(), 0, 1)
                            tmpSnapshotDbm[key] = pickleValue
                        else:
                            # Remove this entry from the DB Snapshot.
                            msg = "Scheduling entry: %s in DB Snapshot " +\
                                  "for disk with ID: %s for removal"
                            logger.debug(msg, diskId, key)
                            # Add entry in the DB Snapshot Deletion DBM marking
                            # the entry for deletion.
                            if (_updateSnapshot(srvObj.getCfg())):
                                snapshotDelDbm.add(key, 1)

                        del tmpFileObj

                # Be friendly and sync the DB file every now and then
                count += 1
                if (count % 100) == 0:
                    if _updateSnapshot(srvObj.getCfg()):
                        snapshotDbm.sync()
                    checkStopJanitorThread(stopEvt)
                    tmpSnapshotDbm.sync()

                #################################################################################################
                #jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath
                #          when iterating at the end of the table that are prone to corrupt the hash table object
                #try:
                #    key, pickleValue = snapshotDbm.next()
                #except:
                #    key = None
                #    snapshotDbm.dbc = None
                #################################################################################################

            # Now, delete entries in the DB Snapshot if there are any scheduled for
            # deletion.

            #################################################################################################
            #jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath
            #          when iterating at the end of the table that are prone to corrupt the hash table object
            #snapshotDelDbm.initKeyPtr()
            #while (True):
            #    key, value = snapshotDelDbm.getNext()
            #    if (not key): break
            for key, value in snapshotDelDbm.iteritems():
                # jagonzal: We need to reformat the values and skip administrative elements #################
                if b'__' in key:
                    continue
                #############################################################################################
                msg = "Removing entry: %s from DB Snapshot for disk with ID: %s"
                logger.debug(msg, key, diskId)
                del snapshotDbm[key]
            #################################################################################################
            del snapshotDelDbm

            logger.debug("Looped over file entries in the DB Snapshot - %s",
                         diskId)
            # End-Loop: Check DB against DB Snapshot. ###########################
            if (_updateSnapshot(srvObj.getCfg())): snapshotDbm.sync()
            tmpSnapshotDbm.sync()

            logger.info(
                "Checked/created/updated DB Snapshot for disk with mount point: %s",
                mtPt)

            #####################################################################
            # Loop over the entries in the DB and compare these against the
            # DB Snapshot.
            #####################################################################
            logger.debug("Loop over the entries in the DB - %s ...", diskId)
            count = 0
            try:
                key, pickleValue = tmpSnapshotDbm.first()
            except:
                key = None
                tmpSnapshotDbm.dbc = None

            #################################################################################################
            #jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath
            #          when iterating at the end of the table that are prone to corrupt the hash table object
            #while (key):
            for key, pickleValue in tmpSnapshotDbm.iteritems():
                #################################################################################################
                value = cPickle.loads(pickleValue)

                # Check if it is an administrative element, if yes add it if needed
                if b"___" in key:
                    if (not snapshotDbm.has_key(key)):
                        snapshotDbm[key] = pickleValue
                else:
                    # Is the file in the DB Snapshot?
                    if (not snapshotDbm.has_key(key)):
                        tmpFileObj = _encFileInfo2Obj(srvObj.getDb(),
                                                      tmpSnapshotDbm, value)
                        if (tmpFileObj is None):
                            continue

                        # Is the file on the disk?
                        complFilename = os.path.normpath(mtPt + "/" +\
                                                         tmpFileObj.getFilename())
                        if (os.path.exists(complFilename)):
                            # Add this entry in the DB Snapshot.
                            if (_updateSnapshot(srvObj.getCfg())):
                                snapshotDbm[key] = pickleValue
                        else:
                            # Remove this entry from the DB (if it is there).
                            _delFileEntry(srvObj.getHostId(), srvObj.getDb(),
                                          tmpFileObj)
                        del tmpFileObj
                    else:
                        # We always update the DB Snapshot to ensure it is
                        # in-sync with the DB entry.
                        if (_updateSnapshot(srvObj.getCfg())):
                            snapshotDbm[key] = pickleValue

                # Be friendly and sync the DB file every now and then
                count += 1
                if (count % 100) == 0:
                    if _updateSnapshot(srvObj.getCfg()):
                        snapshotDbm.sync()
                    checkStopJanitorThread(stopEvt)

                #################################################################################################
                #jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath
                #          when iterating at the end of the table that are prone to corrupt the hash table object
                #try:
                #    key, pickleValue = tmpSnapshotDbm.next()
                #except:
                #    key = None
                #################################################################################################
            logger.debug("Checked DB Snapshot against DB - %s", diskId)
            # End-Loop: Check DB Snapshot against DB. ###########################
            if (_updateSnapshot(srvObj.getCfg())):
                snapshotDbm.sync()

        finally:
            if snapshotDbm:
                snapshotDbm.close()

            if tmpSnapshotDbm:
                tmpSnapshotDbm.close()

    # Check if lost files found.
    logger.debug("Check if there are Lost Files ...")
    noOfLostFiles = lostFileRefsDbm.getCount()
    if (noOfLostFiles):
        statRep = os.path.normpath(tmpDir + "/" + ngasId +\
                                   "_LOST_FILES_NOTIF_EMAIL.txt")
        fo = open(statRep, "w")
        timeStamp = toiso8601()
        tmpFormat = "JANITOR THREAD - LOST FILES DETECTED:\n\n" +\
                    "==Summary:\n\n" +\
                    "Date:                       %s\n" +\
                    "NGAS Host ID:               %s\n" +\
                    "Lost Files:                 %d\n\n" +\
                    "==File List:\n\n"
        fo.write(tmpFormat % (timeStamp, srvObj.getHostId(), noOfLostFiles))

        tmpFormat = "%-32s %-32s %-12s %-80s\n"
        fo.write(tmpFormat %
                 ("Disk ID", "File ID", "File Version", "Expected Path"))
        fo.write(tmpFormat % (32 * "-", 32 * "-", 12 * "-", 80 * "-"))

        # Loop over the files an generate the report.
        lostFileRefsDbm.initKeyPtr()
        while (1):
            key, fileInfoObj = lostFileRefsDbm.getNext()
            if (not key): break
            diskId = fileInfoObj.getDiskId()
            fileId = fileInfoObj.getFileId()
            fileVersion = fileInfoObj.getFileVersion()
            filename = fileInfoObj.getTag()
            fo.write(tmpFormat % (diskId, fileId, fileVersion, filename))
        fo.write("\n\n==END\n")
        fo.close()
        ngamsNotification.notify(srvObj.getHostId(), srvObj.getCfg(),
                                 NGAMS_NOTIF_DATA_CHECK,
                                 "LOST FILE(S) DETECTED", statRep, [], 1,
                                 NGAMS_TEXT_MT, "JANITOR_THREAD_LOST_FILES", 1)
        rmFile(statRep)
    logger.debug("Number of lost files found: %d", noOfLostFiles)

    # Clean up.
    del lostFileRefsDbm
    rmFile(lostFileRefsDbmName + "*")
コード例 #5
0
def _genReport(srvObj, unregistered, diskDic, dbmObjDic, stats):
    """
    Generate the DCC Check Report according to the problems found.

    srvObj:     Reference to instance of ngamsServer object (ngamsServer).

    Returns:    Void.
    """
    # Find out how many inconsistencies were found.
    noOfProbs = 0
    # Errors found.
    for diskId in diskDic.keys():
        noOfProbs += dbmObjDic[diskId][1].getCount()
    # Spurious files on disk.
    unRegFiles = len(unregistered)

    # Generate the report.
    checkTime = time.time() - stats.time_start
    if ((noOfProbs + unRegFiles) or srvObj.getCfg().getDataCheckForceNotif()):

        report = ""
        hdrForm = "%-20s %s\n"
        format = "%-60s %-32s %-9s %s\n"
        separator = 130 * "-" + "\n"

        # Build up the report.
        report = "DATA CHECKING REPORT:\n\n"
        report += hdrForm % ("Date", toiso8601())
        report += hdrForm % ("NGAS Host ID", srvObj.getHostId())
        report += hdrForm % ("Start Time", toiso8601(stats.time_start))
        report += hdrForm % ("Total Time (s)", "%.3f" % checkTime)
        report += hdrForm % ("Total Time (hours)", "%.3f" % (checkTime / 3600))
        report += hdrForm % ("Rate (MB/s)", "%.3f" % stats.check_rate)
        report += hdrForm % ("Files Checked", stats.files_checked)
        report += hdrForm % ("Data Checked (MB)", "%.5f" % stats.mbs_checked)
        report += hdrForm % ("Inconsistencies", str(noOfProbs + unRegFiles))
        report += separator

        # Any discrepancies found?
        if ((not noOfProbs) and (not unRegFiles)):
            report += "No discrepancies found!"

        # Inconsistencies found?
        if (noOfProbs):
            report += "INCONSISTENT FILES FOUND:\n\n"
            report += format % ("Problem Description", "File ID", "Version",
                                "Slot ID:Disk ID")
            report += separator
            for diskId in diskDic.keys():
                errDbm = dbmObjDic[diskId][1].initKeyPtr()
                #################################################################################################
                #jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath
                #          when iterating at the end of the table that are prone to corrupt the hash table object
                #while (1):
                #    fileKey, errInfo = errDbm.getNext()
                #    if (not fileKey): break
                for fileKey, dbVal in errDbm.iteritems():
                    # jagonzal: We need to reformat the values and skip administrative elements #################
                    if (str(fileKey).find("__") != -1): continue
                    errInfo = cPickle.loads(dbVal)
                    #############################################################################################
                    slotDiskId = errInfo[3] + ":" + errInfo[4]
                    report += format % (errInfo[0], errInfo[1], errInfo[2],
                                        slotDiskId)
                #################################################################################################
            report += separator

        # Not registered files found?
        if (unRegFiles):
            repFormat = "%-32s %s\n"
            report += "NOT REGISTERED FILES FOUND ON STORAGE DISKS:\n\n"
            report += repFormat % ("Disk ID:", "Filename:")
            report += separator
            for filename, disk_id in unregistered.items():
                report += repFormat % (disk_id, filename)
            report += separator

        # Send Notification Message if needed (only if disks where checked).
        if (len(diskDic.keys())):
            ngamsNotification.notify(srvObj.getHostId(), srvObj.getCfg(),
                                     NGAMS_NOTIF_DATA_CHECK,
                                     "DATA CHECK REPORT", report, [], 1)

    # Give out the statistics for the checking.
    msg = genLog("NGAMS_INFO_DATA_CHK_STAT", [
        stats.files_checked, unRegFiles, noOfProbs, stats.mbs_checked,
        stats.check_rate, checkTime
    ])
    logger.info(msg)

    # Remove the various DBMs allocated.
    for diskId in diskDic.keys():
        dbmObjDic[diskId][1].cleanUp()
        del dbmObjDic[diskId]
コード例 #6
0
ファイル: clone.py プロジェクト: ICRAR/ngas
def _cloneExec(srvObj,
               cloneListDbmName,
               tmpFilePat,
               targetDiskId,
               reqPropsObj):
    """
    See documentation of ngamsCloneCmd._cloneThread(). This function is
    merely implemented in order to encapsulate the whole process to be able
    to clean up properly when the processing is terminated.
    """
    cloneStatusDbm = None

    emailNotif = 0
    checkChecksum = 1
    if (reqPropsObj):
        if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1
        if (reqPropsObj.hasHttpPar("check")):
            checkChecksum = int(reqPropsObj.getHttpPar("check"))

    # Open clone list DB.
    cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName)

    # We have to get the port numbers of the hosts where the files to be
    # cloned are stored.
    hostInfoDic = {}
    cloneListDbm.initKeyPtr()
    while (1):
        key, fileInfo = cloneListDbm.getNext()
        if (not key): break
        hostInfoDic[fileInfo[1]] = -1
    hostInfoDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(),
                                                       srvObj.getDb(),
                                                       srvObj.getCfg(),
                                                       hostInfoDic.keys())

    # The cloning loop. Loop over the list of files to clone and generate
    # a report with the result.
    if (emailNotif):
        cloneStatusDbmName = tmpFilePat + "_CLONE_STATUS_DB"
        cloneStatusDbm = ngamsDbm.ngamsDbm(cloneStatusDbmName,
                                           cleanUpOnDestr = 0, writePerm = 1)

    successCloneCount = 0
    failedCloneCount  = 0
    abortCloneLoop    = 0
    timeAccu          = 0.0
    key = 0
    while (1):

        clone_start = time.time()

        if (not cloneListDbm.hasKey(str(key))): break
        fileInfo = cloneListDbm.get(str(key))
        key += 1

        # Check if we have permission to run. Otherwise, stop.
        if (not srvObj.run_async_commands): break

        fio = fileInfo[0]
        mtPt = fileInfo[2]
        if (emailNotif):
            tmpFileList = ngamsFileList.\
                          ngamsFileList("FILE_CLONE_STATUS",
                                        "File: " + fio.getFileId() + "/" +\
                                        fio.getDiskId() + "/" +\
                                        str(fio.getFileVersion()))
        hostId = fileInfo[1]
        text = "Cloning file - File ID: %s/%d, on disk " +\
               "with ID: %s on host: %s"
        logger.debug(text, fio.getFileId(), fio.getFileVersion(),
                       fio.getDiskId(), hostId)

        # We generate a local Staging File and archive this.
        stagingFilename = ""
        try:
            # Check if file is marked as bad.
            if (fio.getFileStatus()[0] == "1"):
                errMsg = "File marked as bad - skipping!"
                raise Exception(errMsg)

            if (targetDiskId == ""):
                # Try to find a disk not hosting already a file with that
                # ID + version.
                diskExemptList = [fio.getDiskId()]
                while (1):
                    trgDiskInfo = ngamsDiskUtils.\
                                  findTargetDisk(srvObj.getHostId(),
                                                 srvObj.getDb(),
                                                 srvObj.getCfg(),
                                                 fio.getFormat(),
                                                 1, diskExemptList)
                    # Check if a file with that ID + version is already
                    # stored on the selected Target Disk.
                    if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
                                                fio.getFileId(),
                                                fio.getFileVersion())):
                        # This file is already stored on the given disk.
                        # Add to the exempt list.
                        diskExemptList.append(trgDiskInfo.getDiskId())
                    else:
                        # OK, this disk should be OK, stop looking for a
                        # suitable Target Disk.
                        break
            else:
                try:
                    trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
                                  read(srvObj.getDb(), targetDiskId)
                    slotId = trgDiskInfo.getSlotId()
                    storageSetId = srvObj.getCfg().\
                                   getStorageSetFromSlotId(slotId).\
                                   getStorageSetId()
                    trgDiskInfo.setStorageSetId(storageSetId)
                except Exception:
                    abortCloneLoop = 1
                    raise

            # We don't accept to clone onto the same disk (this would mean
            # overwriting).
            if (trgDiskInfo.getDiskId() == fio.getDiskId()):
                err = "Source and target files are identical"
                msg = "Failed in cloning file with ID: " + fio.getFileId() +\
                      "/Version: " + str(fio.getFileVersion()) +\
                      " on disk with ID: " + fio.getDiskId() +\
                      " on host: " + hostId + ". Reason: " + err
                logger.warning(msg)
                if (emailNotif):
                    tmpFileList.setStatus(NGAMS_FAILURE + ": " + err)
                    tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
                    cloneStatusDbm.addIncKey(tmpFileList)
                failedCloneCount += 1
                continue

            tmpReqPropsObj = ngamsReqProps.ngamsReqProps()
            tmpReqPropsObj.setMimeType(fio.getFormat())
            stagingFilename = ngamsHighLevelLib.\
                              genStagingFilename(srvObj.getCfg(),
                                                 tmpReqPropsObj,
                                                 trgDiskInfo, fio.getFileId())
            # Receive the data into the Staging File using the urllib.
            if (srvObj.getHostId() != hostId):
                # Example: http://host:7777/RETRIEVE?file_id=id&file_version=1
                ipAddress = hostInfoDic[hostId].getIpAddress()
                portNo = hostInfoDic[hostId].getSrvPort()
                fileUrl = "http://" + ipAddress + ":" + str(portNo) +\
                          "/RETRIEVE?" + "file_id=" + fio.getFileId() +\
                          "&file_version=" + str(fio.getFileVersion())
                # If a specific Disk ID for the source file is given, append
                # this.
                if (fio.getDiskId()):
                    fileUrl += "&disk_id=%s" % fio.getDiskId()

                # Check if host is suspended, if yes, wake it up.
                if (srvObj.getDb().getSrvSuspended(hostId)):
                    logger.debug("Clone Request - Waking up suspended " +\
                         "NGAS Host: %s", hostId)
                    ngamsSrvUtils.wakeUpHost(srvObj, hostId)
            else:
                fileUrl = "file:" + mtPt + "/" + fio.getFilename()
            logger.debug("Receiving file via URI: %s into staging filename: %s",
                         fileUrl, stagingFilename)
            # We try up to 5 times to retrieve the file in case a problem is
            # encountered during cloning.
            for attempt in range(5):
                try:
                    filename, headers = urlrequest.urlretrieve(fileUrl, stagingFilename)
                    _checkFile(srvObj, fio, stagingFilename, headers,
                               checkChecksum)
                    # If we get to this point the transfer was (probably) OK.
                    break
                except Exception as e:
                    rmFile(stagingFilename)
                    errMsg = "Problem occurred while cloning file "+\
                             "via URL: " + fileUrl + " - Error: " + str(e)
                    if (attempt < 4):
                        errMsg += " - Retrying in 5s ..."
                        logger.error(errMsg)
                        time.sleep(5)
                    else:
                        raise Exception(errMsg)

            # We simply copy the file into the same destination as the
            # source file (but on another disk).
            targPathName  = os.path.dirname(fio.getFilename())
            targFilename  = os.path.basename(fio.getFilename())
            complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\
                                             "/" + targPathName)
            checkCreatePath(complTargPath)
            complFilename = os.path.normpath(complTargPath + "/"+targFilename)
            mvTime = mvFile(stagingFilename, complFilename)
            ngamsLib.makeFileReadOnly(complFilename)

            # Update status for new file in the DB.
            newFileInfo = fio.clone().setDiskId(trgDiskInfo.getDiskId()).\
                          setCreationDate(getFileCreationTime(complFilename))
            fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
                                                 fio.getFileId(),
                                                 fio.getFileVersion())
            newFileInfo.write(srvObj.getHostId(), srvObj.getDb())

            # Update status for the Target Disk in DB + check if the disk is
            # completed.
            if (fileExists): mvTime = 0
            dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\
                               setDiskId(trgDiskInfo.getDiskId()).\
                               setFileExists(fileExists).\
                               setFileSize(fio.getFileSize()).setIoTime(mvTime)
            ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj)
            ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId())

            # Update the clone file status list.
            if (emailNotif):
                tmpFileList.setStatus(NGAMS_SUCCESS)
                tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
                tmpFileList.addFileInfoObj(newFileInfo.setTag("TARGET_FILE"))
                cloneStatusDbm.addIncKey(tmpFileList)
            successCloneCount += 1

            # If running as a cache archive, update the Cache New Files DBM
            # with the information about the new file.
            if (srvObj.getCachingActive()):
                diskId   = trgDiskInfo.getDiskId()
                fileId   = fio.getFileId()
                fileVer  = fio.getFileVersion()
                filename = fio.getFilename()
                ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId,
                                                            fileId, fileVer,
                                                            filename)

            # Generate a confirmation log entry.
            cloneTime = time.time() - clone_start
            timeAccu += cloneTime
            msg = genLog("NGAMS_INFO_FILE_CLONED",
                         [fio.getFileId(), fio.getFileVersion(),
                          fio.getDiskId(), hostId])
            msg = msg + ". Time: %.3fs. Total time: %.3fs." %\
                  (cloneTime, timeAccu)
            logger.info(msg, extra={'to_syslog': True})
        except Exception as e:
            cloneTime = time.time() - clone_start
            timeAccu += cloneTime
            errMsg = genLog("NGAMS_ER_FILE_CLONE_FAILED",
                            [fio.getFileId(), fio.getFileVersion(),
                             fio.getDiskId(), hostId, str(e)])
            if (abortCloneLoop):
                logger.error(errMsg, extra={'to_syslog': True})
                return
            else:
                logger.warning(errMsg)
                if (emailNotif):
                    tmpFileList.setStatus(NGAMS_FAILURE + ": Error: " + errMsg)
                    tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
                    cloneStatusDbm.addIncKey(tmpFileList)
                failedCloneCount += 1

            # Delete Staging File if already created.
            if ((stagingFilename != "") and (os.path.exists(stagingFilename))):
                rmFile(stagingFilename)

        # Calculate time statistics.
        if (reqPropsObj):
            ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj.\
                                                   incActualCount(1), timeAccu)

    # Final update of the Request Status.
    if (reqPropsObj):
        complPercent = (100.0 * (float(reqPropsObj.getActualCount()) /
                                 float(reqPropsObj.getExpectedCount())))
        reqPropsObj.setCompletionPercent(complPercent, 1)
        reqPropsObj.setCompletionTime(1)
        srvObj.updateRequestDb(reqPropsObj)

    # Send Clone Report with list of files cloned to a possible
    # requestor(select) of this.
    totFiles = (successCloneCount + failedCloneCount)
    if (emailNotif):
        xmlStat = 0
        # TODO: Generation of XML status report is disabled since we cannot
        #       handle for the moment XML documents with 1000s of elements.
        if (xmlStat):
            cloneStatusFileList = ngamsFileList.\
                                  ngamsFileList("FILE_CLONING_STATUS_REPORT",
                                                "File Cloning Status Report")
            fileCount = 0
            while (fileCount < cloneStatusDbm.getCount()):
                tmpFileList = cloneStatusDbm.get(str(fileCount))
                cloneStatusFileList.addFileListObj(tmpFileList)

            # Make overall status.
            cloneStatusFileList.setStatus("SUCCESS: " +\
                                          str(successCloneCount) +\
                                          ", FAILURE: " +\
                                          str(failedCloneCount) +\
                                          ", NOT DONE: " +\
                                          str(len(cloneStatusFileList) -\
                                              successCloneCount -\
                                              failedCloneCount))
            status = srvObj.genStatus(NGAMS_SUCCESS,
                                      "CLONE command status report").\
                                      addFileList(cloneStatusFileList)
            statRep = status.genXmlDoc(0, 0, 0, 1, 0)
            statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep)
            mimeType = NGAMS_XML_MT
        else:
            # Generate a 'simple' ASCII report.
            statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
            fo = open(statRep, "w")
            if (reqPropsObj.hasHttpPar("disk_id")):
                diskId = reqPropsObj.getHttpPar("disk_id")
            else:
                diskId = "-----"
            if (reqPropsObj.hasHttpPar("file_id")):
                fileId = reqPropsObj.getHttpPar("file_id")
            else:
                fileId = "-----"
            if (reqPropsObj.hasHttpPar("file_version")):
                fileVersion = reqPropsObj.getHttpPar("file_version")
            else:
                fileVersion = "-----"
            tmpFormat = "CLONE STATUS REPORT:\n\n" +\
                        "==Summary:\n\n" +\
                        "Date:                       %s\n" +\
                        "NGAS Host:                  %s\n" +\
                        "Disk ID:                    %s\n" +\
                        "File ID:                    %s\n" +\
                        "File Version:               %s\n" +\
                        "Total Number of Files:      %d\n" +\
                        "Number of Cloned Files:     %d\n" +\
                        "Number of Failed Files:     %d\n" +\
                        "Total processing time (s):  %.3f\n" +\
                        "Handling time per file (s): %.3f\n\n" +\
                        "==File List:\n\n"
            fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), diskId, fileId,
                                  str(fileVersion), totFiles,
                                  successCloneCount, failedCloneCount,
                                  timeAccu, (timeAccu / totFiles)))
            tmpFormat = "%-70s %-70s %-7s\n"
            fo.write(tmpFormat % ("Source File", "Target File", "Status"))
            fo.write(tmpFormat % (70 * "-", 70 * "-", 7 * "-"))
            key = 1
            while (1):
                if (not cloneStatusDbm.hasKey(str(key))): break
                tmpFileList = cloneStatusDbm.get(str(key))
                key += 1
                srcFileObj = tmpFileList.getFileInfoObjList()[0]
                srcFile = "%s/%s/%d" % (srcFileObj.getDiskId(),
                                        srcFileObj.getFileId(),
                                        srcFileObj.getFileVersion())
                if (tmpFileList.getStatus() == NGAMS_SUCCESS):
                    trgFileObj = tmpFileList.getFileInfoObjList()[1]
                    trgFile = "%s/%s/%d" % (trgFileObj.getDiskId(),
                                            trgFileObj.getFileId(),
                                            trgFileObj.getFileVersion())
                else:
                    trgFile = "-----"
                fo.write(tmpFormat % (srcFile,trgFile,tmpFileList.getStatus()))
            fo.write(149 * "-")
            fo.write("\n\n==END\n")
            fo.close()
            mimeType = NGAMS_TEXT_MT

        # Send out the status report.
        emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",")
        attachmentName = "CloneStatusReport"
        if (reqPropsObj.hasHttpPar("disk_id")):
            attachmentName += "-" + reqPropsObj.getHttpPar("disk_id")
        if (reqPropsObj.hasHttpPar("file_id")):
            attachmentName += "-" + reqPropsObj.getHttpPar("file_id")
        if (reqPropsObj.hasHttpPar("file_version")):
            attachmentName += "-" + reqPropsObj.getHttpPar("file_version")
        ngamsNotification.notify(srvObj.host_id, srvObj.cfg, NGAMS_NOTIF_INFO,
            "CLONE STATUS REPORT", statRep, recList=emailAdrList, force=1,
            contentType=mimeType, attachmentName=attachmentName)
        del cloneStatusDbm
        rmFile(cloneStatusDbmName + "*")
        rmFile(statRep)

    if (cloneListDbm): del cloneListDbm
    rmFile(cloneListDbmName + "*")
    logger.info("_cloneExec(). Total time: %.3fs. Average time per file: %.3fs.",
                timeAccu, (timeAccu / totFiles))
コード例 #7
0
def _create_remote_subscriptions(srvObj, stop_evt):
    """
    Creates subscriptions in remote servers so they push their data into
    ours.
    """

    subscriptions_in_cfg = srvObj.cfg.getSubscriptionsDic()
    subscriptions = [v for _, v in subscriptions_in_cfg.items()]
    subscriptions_created = 0

    def mark_as_active(s):
        # Removes s from list of subscriptions pending creation, and
        # ensures the corresponding UNSUBSCRIBE command will be called
        # at server shutdown
        srvObj.getSubscrStatusList().append(s)
        subscriptions.remove(s)

    while True:

        # Done with all of them
        if not subscriptions:
            break

        # Iterate over copy, since we modify the original inside the loop
        for subscrObj in list(subscriptions):

            if stop_evt.is_set():
                logger.info("Terminating Subscriber thread ...")
                return

            our_host, our_port = srvObj.get_self_endpoint()
            subs_host, subs_port = subscrObj.getHostId(), subscrObj.getPortNo()

            # Not subscribing to ourselves
            if srvObj.is_it_us(subs_host, subs_port):
                logger.warning("Skipping subscription to %s:%d because that's us", subs_host, subs_port)
                continue

            # Because properly supporting the "Command" configuration mechanism
            # still requires some more work, we prefer the "SubscriberUrl"
            # attribute as the main source of URL information.
            # We still support "Command", but with the following caveats:
            #  * TODO: include reverse proxy information when we add support
            #  * TODO: hardcoded http will need to be changed when we add support
            #          for https
            #  * TODO: fails with IpAddress == '0.0.0.0'
            url = subscrObj.getUrl()
            if not url:
                url = 'http://%s:%d/%s' % (our_host, our_port, subscrObj.command or 'QARCHIVE')
            logger.info("Creating subscription to %s:%d with url=%s", subs_host, subs_port, url)

            pars = [["subscr_id", subscrObj.getId()],
                    ["priority", subscrObj.getPriority()],
                    ["url",      url],
                    ["start_date", toiso8601(local=True)]]
            if subscrObj.getFilterPi():
                pars.append(["filter_plug_in", subscrObj.getFilterPi()])
            if subscrObj.getFilterPiPars():
                pars.append(["plug_in_pars", subscrObj.getFilterPiPars()])

            try:
                # Issue SUBSCRIBE command
                resp = ngamsHttpUtils.httpGet(subs_host, subs_port, cmd=NGAMS_SUBSCRIBE_CMD, pars=pars)
                with contextlib.closing(resp):
                    stat = ngamsStatus.to_status(resp, subscrObj.getHostId(), NGAMS_SUBSCRIBE_CMD)

                if stat.getStatus() != NGAMS_SUCCESS:
                    if stat.http_status == 409:
                        short_msg = "Different subscription with ID '%s' already exists, giving up"
                        long_msg = (
                            "NGAS attempted to create an automatic subscription "
                            "with ID=%s to obtain data from %s:%d, but the remote server "
                            "already has a subscription registered with the same ID, "
                            "but different details.\n\n"
                            "Instead of retrying to create this subscription over and over, "
                            "this server will give up now. To fix this either remove "
                            "the remote subscription, or change the ID of the subscription to be created "
                            "in the local server configuration."
                        )
                        logger.error(short_msg, subscrObj.getId())
                        ngamsNotification.notify(
                            srvObj.host_id, srvObj.cfg, NGAMS_NOTIF_ERROR,
                            "Automatic subscription cannot be created",
                            long_msg % (subscrObj.getId(), subs_host, subs_port),
                            force=True)
                        mark_as_active(subscrObj)
                        continue
                    else:
                        msg = "Unsuccessful NGAS XML response. Status: %s, message: %s. Will try again later"
                        logger.warning(msg, stat.getStatus(), stat.getMessage())
                        continue

                subscriptions_created += 1
                logger.info("Successfully subscribed to %s:%d with url=%s", subs_host, subs_port, subscrObj.getUrl())
                mark_as_active(subscrObj)

            except:
                logger.exception("Error while adding subscription, will try later")

        if stop_evt.wait(10):
            return

    if subscriptions_created:
        logger.info("Successfully established %d Subscription(s)", subscriptions_created)
    else:
        logger.info("No Subscriptions established")
コード例 #8
0
def _registerExec(srvObj,
                  fileListDbmName,
                  tmpFilePat,
                  diskInfoDic,
                  reqPropsObj=None):
    """
    Register the files listed in the File List DBM (ngamsDbm), which
    match the mime-type(s) either specified in the 'mimeType' parameter,
    or if this is not specified, which match all the mime-types specified
    in the configuration file.

    When the registration procedure has been executed, the function sends
    an Email Notification message indicating which files were registered
    if the HTTP parameter 'notif_email' is given.

    The functions creates a File Info Objects per file handled and
    writes this in a temporary file, which is a DBM file. The
    keys in this DB is simply the file number in the sequence of files
    handled, pointing to a pickled ngamsFileInfo object.

    Each of the File Info Objects indicates if the file was registered or not.
    This is done by setting the tag of the File Info Object to one of the
    following values:

      REGISTERED:  The file was successfully registered in the NGAS DB

      FAILED:      The file was selected for registration but could not
                   be properly registered because of inconsistencies.
                   The status will be of the format: 'FAILED[: <reason>]'.

      REJECTED:    A file found under the specified path directory was
                   not accepted for cloning, usually because the mime-type
                   was not correct. The status will be of the format:
                   'REJECTED[: <reason>]'.

    Note, that registration is comparable to archiving of files. For that
    reason a DAPI must be provided for each type of file that should be
    registered. If this is not fullfilled, the file registration will
    fail.

    Only files stored on one of the NGAS disks configured in the
    configuration file, are considered. Files stored in other locations
    are ignored.

    srvObj:          Instance of NG/AMS Server object (ngamsServer).

    fileListDbmName: Name of a DBM containing the information
                     about the files to be registered. Each element in the
                     list is referred to by a key, which is a number.
                     These points to a pickled list for each file containing
                     the following information:

                       [<Filename>, <Disk ID>, <Mime-Type>]

                     The information for each disk concerned is also contained
                     in the DB referred to by its Disk ID and by its mount
                     point. The data is a pickled instance of the ngamsDiskInfo
                     class (string).

    tmpFilePat:      Pattern for temporary files used during the registration
                     process (string).

    diskInfoDic:     Dictionary with Disk IDs as keys pointing to the info
                     about the disk (dictionary/ngamsDiskInfo).

    reqPropsObj:     If an NG/AMS Request Properties Object is given, the
                     Request Status will be updated as the request is carried
                     out (ngamsReqProps).

    Returns:         Void.
    """
    emailNotif = 0
    if (reqPropsObj):
        if (reqPropsObj.hasHttpPar("notif_email")):
            emailNotif = 1

    # Create the temporary BSD DB to contain the information for the
    # Email Notification Message.
    if (emailNotif):
        regDbmName = tmpFilePat + "_NOTIF_EMAIL"
        regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm=1)

    # Open the DBM containing the list of files to (possibly) register.
    fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1)

    # Want to parse files in alphabetical order.
    # TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting.
    tmpFileList = tmpFilePat + "_FILE_LIST"
    rmFile(tmpFileList)
    with open(tmpFileList, "wb") as fo:
        fileListDbm.initKeyPtr()
        while (1):
            dbmKey, fileInfo = fileListDbm.getNext()
            if (not dbmKey): break
            fo.write(dbmKey + b"\n")
    sortFileList = tmpFilePat + "_SORT_FILE_LIST"
    rmFile(sortFileList)
    shellCmd = "sort %s > %s" % (tmpFileList, sortFileList)
    stat, out, err = ngamsCore.execCmd(shellCmd)
    if (stat != 0):
        raise Exception("Error executing command: %s. Error: %s, %s" %\
              (shellCmd, str(out), str(err)))
    rmFile(tmpFileList)

    # Go through each file in the list, check if the mime-type is among the
    # ones, which apply for registration. If yes try to register the file
    # by invoking the corresponding DAPI on the file.
    fileRegCount = 0
    fileFailCount = 0
    fileRejectCount = 0
    regTimeAccu = 0.0
    fileCount = 0
    fo = open(sortFileList)
    run = 1
    while (run):
        reg_start = time.time()
        dbmKey = fo.readline()
        if (dbmKey.strip() == ""):
            run = 0
            continue
        fileInfo = fileListDbm.get(dbmKey[0:-1])
        filename = fileInfo[0]
        diskId = fileInfo[1]
        mimeType = fileInfo[2]

        # Register the file. Check first, that exactly this file is
        # not already registered. In case it is, the file will be rejected.
        regPi = srvObj.getCfg().register_plugins[mimeType]
        logger.debug("Plugin found for %s: %s", mimeType, regPi)
        params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars)
        tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\
                         setMimeType(mimeType).\
                         setStagingFilename(filename).\
                         setTargDiskInfo(diskInfoDic[diskId]).\
                         setHttpMethod(NGAMS_HTTP_GET).\
                         setCmd(NGAMS_REGISTER_CMD).\
                         setSize(os.path.getsize(filename)).\
                         setFileUri(filename).\
                         setNoReplication(1)

        tmpFileObj = ngamsFileInfo.ngamsFileInfo()
        try:
            # Invoke Registration Plug-In.
            piName = regPi.name
            plugInMethod = loadPlugInEntryPoint(piName)
            piRes = plugInMethod(srvObj, tmpReqPropsObj, params)
            del tmpReqPropsObj

            # Check if this file is already registered on this disk. In case
            # yes, it is not registered again.
            files = srvObj.db.getFileSummary1(srvObj.getHostId(),
                                              [piRes.getDiskId()],
                                              [piRes.getFileId()])
            fileRegistered = 0
            for tmpFileInfo in files:
                tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT]
                tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME]
                tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\
                                                    tmpFilename)
                if (tmpComplFilename == filename):
                    fileRegistered = 1
                    break

            if (fileRegistered):
                fileRejectCount += 1
                tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\
                             "and path: %s is already registered on disk " +\
                             "with Disk ID: %s"
                tmpMsg = tmpMsgForm % (piRes.getFileId(),
                                       piRes.getFileVersion(), filename,
                                       piRes.getDiskId())
                logger.warning(tmpMsg + ". File is not registered again.")
                if (emailNotif):
                    tmpFileObj.\
                                 setDiskId(diskId).setFilename(filename).\
                                 setTag(tmpMsg)
                    regDbm.addIncKey(tmpFileObj)
                if (reqPropsObj):
                    reqPropsObj.incActualCount(1)
                    ngamsHighLevelLib.stdReqTimeStatUpdate(
                        srvObj, reqPropsObj, regTimeAccu)
                regTimeAccu += time.time() - reg_start
                fileCount += 1
                continue

            # Calculate checksum. We maintain the old name for backwards
            # compatibility
            crc_variant = srvObj.cfg.getCRCVariant()
            if crc_variant == ngamsFileUtils.CHECKSUM_CRC32_INCONSISTENT:
                crc_variant = 'ngamsGenCrc32'
            checksum = ngamsFileUtils.get_checksum(65536, filename,
                                                   crc_variant) or ''

            # Move file and update information about file in the NGAS DB.
            mvFile(filename, piRes.getCompleteFilename())
            ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum,
                                               crc_variant)
            ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes)
            ngamsLib.makeFileReadOnly(piRes.getCompleteFilename())

            if (emailNotif):
                uncomprSize = piRes.getUncomprSize()
                ingestDate = time.time()
                creDateSecs = getFileCreationTime(filename)
                tmpFileObj.\
                             setDiskId(diskId).\
                             setFilename(filename).\
                             setFileId(piRes.getFileId()).\
                             setFileVersion(piRes.getFileVersion()).\
                             setFormat(piRes.getFormat()).\
                             setFileSize(piRes.getFileSize()).\
                             setUncompressedFileSize(uncomprSize).\
                             setCompression(piRes.getCompression()).\
                             setIngestionDate(ingestDate).\
                             setIgnore(0).\
                             setChecksum(checksum).\
                             setChecksumPlugIn(crc_variant).\
                             setFileStatus(NGAMS_FILE_STATUS_OK).\
                             setCreationDate(creDateSecs).\
                             setTag("REGISTERED")
            fileRegCount += 1

            # If running as a cache archive, update the Cache New Files DBM
            # with the information about the new file.
            if (srvObj.getCachingActive()):
                fileVer = fio.getFileVersion()
                ngamsCacheControlThread.addEntryNewFilesDbm(
                    srvObj, diskId, piRes.getFileId(), fileVer, filename)

            # Generate a confirmation log entry.
            msg = genLog("NGAMS_INFO_FILE_REGISTERED", [
                filename,
                piRes.getFileId(),
                piRes.getFileVersion(),
                piRes.getFormat()
            ])
            time.sleep(0.005)
            regTime = time.time() - reg_start
            msg = msg + ". Time: %.3fs." % (regTime)
            logger.info(msg, extra={'to_syslog': 1})
        except Exception as e:
            errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)])
            logger.error(errMsg)
            if (emailNotif):
                tmpFileObj.\
                             setDiskId(diskId).setFilename(filename).\
                             setTag(errMsg)
            fileFailCount += 1
            regTime = time.time() - reg_start
            # TODO (rtobar, 2016-01): Why don't we raise an exception here?
            #      Otherwise the command appears as successful on the
            #      client-side

        # Add the file information in the registration report.
        if (emailNotif): regDbm.addIncKey(tmpFileObj)

        # Update request status time information.
        regTimeAccu += regTime
        if (reqPropsObj):
            reqPropsObj.incActualCount(1)
            ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
                                                   regTimeAccu)
        fileCount += 1
    fo.close()
    rmFile(sortFileList)
    if (emailNotif): regDbm.sync()
    del fileListDbm
    rmFile(fileListDbmName + "*")

    # Final update of the Request Status.
    if (reqPropsObj):
        if (reqPropsObj.getExpectedCount() and reqPropsObj.getActualCount()):
            complPercent = (100.0 * (float(reqPropsObj.getActualCount()) /
                                     float(reqPropsObj.getExpectedCount())))
        else:
            complPercent = 100.0
        reqPropsObj.setCompletionPercent(complPercent, 1)
        reqPropsObj.setCompletionTime(1)
        srvObj.updateRequestDb(reqPropsObj)

    # Send Register Report with list of files cloned to a possible
    # requestor(select) of this.
    if (emailNotif):
        xmlStat = 0
        if (xmlStat):
            # Create an instance of the File List Class, used to store the
            # Registration Report.
            regStat = ngamsFileList.\
                      ngamsFileList("FILE_REGISTRATION_STATUS",
                                    "Status report for file " +\
                                    "registration")

            # Loop over the file objects in the BSD DB and add these
            # in the status object.
            regDbm.initKeyPtr()
            while (1):
                key, tmpFileObj = regDbm.getNext()
                if (not key): break
                regStat.addFileInfoObj(tmpFileObj)

            # Set overall status of registration procedure.
            regStat.setStatus("Files Found: " + str(fileCount + 1) + ", "+\
                              "Files Registered: " + str(fileRegCount) +\
                              ", " +\
                              "Files Failed: " + str(fileFailCount) + ", " +\
                              "Files Rejected: " + str(fileRejectCount))
            status = srvObj.genStatus(NGAMS_SUCCESS,
                                      "REGISTER command status report").\
                                      addFileList(regStat)
            statRep = status.genXmlDoc()
            statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep)
            mimeType = NGAMS_XML_MT
        else:
            # Generate a 'simple' ASCII report.
            statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
            fo = open(statRep, "w")
            if (reqPropsObj):
                path = reqPropsObj.getHttpPar("path")
            else:
                path = "-----"
            if (fileCount):
                timePerFile = (regTimeAccu / fileCount)
            else:
                timePerFile = 0
            tmpFormat = "REGISTER STATUS REPORT:\n\n" +\
                        "==Summary:\n\n" +\
                        "Date:                       %s\n" +\
                        "NGAS Host:                  %s\n" +\
                        "Search Path:                %s\n" +\
                        "Total Number of Files:      %d\n" +\
                        "Number of Registered Files: %d\n" +\
                        "Number of Failed Files:     %d\n" +\
                        "Number of Rejected Files:   %d\n" +\
                        "Total processing time (s):  %.3f\n" +\
                        "Handling time per file (s): %.3f\n\n" +\
                        "==File List:\n\n"
            fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), path,
                                  fileCount, fileRegCount, fileFailCount,
                                  fileRejectCount, regTimeAccu, timePerFile))
            tmpFormat = "%-80s %-32s %-3s %-10s\n"
            fo.write(tmpFormat %\
                     ("Filename", "File ID", "Ver", "Status"))
            fo.write(tmpFormat % (80 * "-", 32 * "-", 3 * "-", 10 * "-"))
            regDbm.initKeyPtr()
            while (1):
                key, tmpFileObj = regDbm.getNext()
                if (not key): break
                mtPt = diskInfoDic[tmpFileObj.getDiskId()].getMountPoint()
                filename = os.path.normpath(mtPt + "/" +\
                                            tmpFileObj.getFilename())
                line = tmpFormat %\
                       (filename, tmpFileObj.getFileId(),
                        str(tmpFileObj.getFileVersion()),
                        tmpFileObj.getTag())
                fo.write(line)

            fo.write(128 * "-")
            fo.write("\n\n==END\n")
            fo.close()
            mimeType = NGAMS_TEXT_MT

        # Send out the status report.
        emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",")
        attachmentName = "RegisterStatusReport"
        if (reqPropsObj.hasHttpPar("path")):
            attachmentName += "-" + reqPropsObj.getHttpPar("path").\
                              replace("/", "_")
        ngamsNotification.notify(srvObj.host_id,
                                 srvObj.cfg,
                                 NGAMS_NOTIF_INFO,
                                 "REGISTER STATUS REPORT",
                                 statRep,
                                 recList=emailAdrList,
                                 force=1,
                                 contentType=mimeType,
                                 attachmentName=attachmentName)
        del regDbm
        rmFile(regDbmName + "*")
        rmFile(statRep)

    # Generate final status log + exit.
    if (fileCount > 0):
        timePerFile = (regTimeAccu / fileCount)
    else:
        timePerFile = 0.0

    msg = "Registration procedure finished processing Register Request - " + \
          "terminating. Files handled: %d. Total time: %.3fs. " + \
          "Average time per file: %.3fs."
    logger.debug(msg, fileCount, regTimeAccu, timePerFile)
コード例 #9
0
                    "Lost Files:                 %d\n\n" +\
                    "==File List:\n\n"
        fo.write(tmpFormat % (timeStamp, srvObj.getHostId(), noOfLostFiles))

        tmpFormat = "%-32s %-32s %-12s %-80s\n"
        fo.write(tmpFormat % ("Disk ID", "File ID", "File Version",
                              "Expected Path"))
        fo.write(tmpFormat % (32 * "-", 32 * "-", 12 * "-", 80 * "-"))

        # Loop over the files an generate the report.
        lostFileRefsDbm.initKeyPtr()
        while (1):
            key, fileInfoObj = lostFileRefsDbm.getNext()
            if (not key): break
            diskId      = fileInfoObj.getDiskId()
            fileId      = fileInfoObj.getFileId()
            fileVersion = fileInfoObj.getFileVersion()
            filename    = fileInfoObj.getTag()
            fo.write(tmpFormat % (diskId, fileId, fileVersion, filename))
        fo.write("\n\n==END\n")
        fo.close()
        ngamsNotification.notify(srvObj.getHostId(), srvObj.getCfg(), NGAMS_NOTIF_DATA_CHECK,
                                 "LOST FILE(S) DETECTED", statRep,
                                 [], 1, NGAMS_TEXT_MT,
                                 "JANITOR_THREAD_LOST_FILES", 1)
        rmFile(statRep)
    logger.debug("Number of lost files found: %d", noOfLostFiles)

    # Clean up.
    del lostFileRefsDbm
    rmFile(lostFileRefsDbmName + "*")