def _cloneExplicit(srvObj, reqPropsObj, diskId, fileId, fileVersion, targetDiskId): """ Execute CLONE Command, where the source Disk ID, File ID and File Version are specified. Is much faster than a normal CLONE Command when an explicit file is specified. srvObj: Reference to instance of Server Object (ngamsServer). fileInfoObj: File info object with info about the file diskId: ID of disk hosting the file to be cloned (string). fileId: ID of file to clone (string). fileVersion: Version of file to clone (integer). targetDiskId: ID of target disk (string). Returns: Void. """ T = TRACE(1) # Resolve the location of the file to clone. location, hostId, ipAddress, portNo, mountPoint, filename,\ fileVersion, mimeType =\ ngamsFileUtils.quickFileLocate(srvObj, reqPropsObj, fileId, diskId=diskId, fileVersion=fileVersion) # Read also the entire file info (unfortunately). srcFileInfo = ngamsFileInfo.ngamsFileInfo().read(srvObj.getHostId(), srvObj.getDb(), fileId, fileVersion, diskId) # Determine target disk. if (targetDiskId == ""): # Try to find a disk not hosting already a file with that # ID + version. diskExemptList = [diskId] while (1): trgDiskInfo = ngamsDiskUtils.\ findTargetDisk(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), mimeType, 1, diskExemptList) # Check if a file with that ID + version is already # stored on the selected Target Disk. if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fileId, fileVersion)): # This file is already stored on the given disk. # Add to the exempt list. diskExemptList.append(trgDiskInfo.getDiskId()) else: # OK, this disk should be OK, stop looking for a # suitable Target Disk. break else: trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\ read(srvObj.getDb(), targetDiskId) slotId = trgDiskInfo.getSlotId() storageSetId = srvObj.getCfg().getStorageSetFromSlotId(slotId).\ getStorageSetId() trgDiskInfo.setStorageSetId(storageSetId) # Don't accept to clone onto the same disk (this would meann overwriting). if (trgDiskInfo.getDiskId() == diskId): err = "Source and target files are identical" msg = "Failed in cloning file with ID: " + fileId +\ "/Version: " + str(fileVersion) +\ " on disk with ID: " + diskId +\ " on host: " + hostId + ". Reason: " + err raise Exception, msg # Receive the file into the staging filename. tmpReqPropsObj = ngamsReqProps.ngamsReqProps() tmpReqPropsObj.setMimeType(mimeType) stagingFilename = ngamsHighLevelLib.genStagingFilename( srvObj.getCfg(), tmpReqPropsObj, trgDiskInfo, fileId) try: quickLocation = False if (reqPropsObj.hasHttpPar("quick")): quickLocation = int(reqPropsObj.getHttpPar("quick")) # Receive the data into the Staging File using the urllib. if (srvObj.getHostId() != hostId): # Example: http://host:7777/RETRIEVE?disk_id=%s&" # file_id=id&file_version=1 fileUrl = "http://%s:%s/RETRIEVE?disk_id=%s&file_id=%s&" +\ "file_version=%s" fileUrl = fileUrl % (ipAddress, str(portNo), diskId, fileId, str(fileVersion)) # If CLONE?quick specified, we try to retrieve the file via the # RETRIEVE?quick_location method. quickFileUrl = fileUrl if (reqPropsObj.hasHttpPar("quick")): if (int(reqPropsObj.getHttpPar("quick"))): quickFileUrl = fileUrl + "&quick_location=1" # Check if host is suspended, if yes, wake it up. if (srvObj.getDb().getSrvSuspended(hostId)): logger.debug("Clone Request - Waking up suspended " +\ "NGAS Host: %s", hostId) ngamsSrvUtils.wakeUpHost(srvObj, hostId) else: # TODO: a time-bomb waiting to explode.... fileUrl = "file:" + mtPt + "/" + filename logger.debug("Receiving file via URI: %s into staging filename: %s", fileUrl, stagingFilename) # We try up to 5 times to retrieve the file in case a problem is # encountered during cloning. for attempt in range(5): try: if (attempt == 0): filename, headers = urllib.urlretrieve( quickFileUrl, stagingFilename) else: filename, headers = urllib.urlretrieve( fileUrl, stagingFilename) _checkFile(srvObj, srcFileInfo, stagingFilename, headers, True) # If we get to this point the transfer was (probably) OK. break except Exception, e: rmFile(stagingFilename) errMsg = "Problem occurred while cloning file "+\ "via URL: " + fileUrl + " - Error: " + str(e) if (attempt < 4): errMsg += " - Retrying in 5s ..." logger.error(errMsg) time.sleep(0.5) else: raise Exception, errMsg # We simply copy the file into the same destination as the # source file (but on another disk). targPathName = os.path.dirname(srcFileInfo.getFilename()) targFilename = os.path.basename(srcFileInfo.getFilename()) complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\ "/" + targPathName) checkCreatePath(complTargPath) complFilename = os.path.normpath(complTargPath + "/" + targFilename) mvTime = mvFile(stagingFilename, complFilename) ngamsLib.makeFileReadOnly(complFilename) # Update status for new file in the DB. newFileInfo = srcFileInfo.clone().setDiskId(trgDiskInfo.getDiskId()).\ setCreationDate(getFileCreationTime(complFilename)) fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fileId, fileVersion) newFileInfo.write(srvObj.getHostId(), srvObj.getDb()) # Update status for the Target Disk in DB + check if the disk is # completed. if (fileExists): mvTime = 0 dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\ setDiskId(trgDiskInfo.getDiskId()).\ setFileExists(fileExists).\ setFileSize(srcFileInfo.getFileSize()).\ setIoTime(mvTime) ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj) ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId()) # If running as a cache archive, update the Cache New Files DBM # with the information about the new file. if (srvObj.getCachingActive()): ngamsCacheControlThread.addEntryNewFilesDbm( srvObj, diskId, fileId, fileVersion, filename) # Generate a confirmation log entry. msg = genLog("NGAMS_INFO_FILE_CLONED", [fileId, fileVersion, diskId, hostId]) logger.info(msg, extra={'to_syslog': True})
def _cloneExec(srvObj, cloneListDbmName, tmpFilePat, targetDiskId, reqPropsObj): """ See documentation of ngamsCloneCmd._cloneThread(). This function is merely implemented in order to encapsulate the whole process to be able to clean up properly when the processing is terminated. """ T = TRACE(1) cloneListDbm = cloneStatusDbm = None emailNotif = 0 checkChecksum = 1 if (reqPropsObj): if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1 if (reqPropsObj.hasHttpPar("check")): checkChecksum = int(reqPropsObj.getHttpPar("check")) # Open clone list DB. cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName) # We have to get the port numbers of the hosts where the files to be # cloned are stored. hostInfoDic = {} cloneListDbm.initKeyPtr() while (1): key, fileInfo = cloneListDbm.getNext() if (not key): break hostInfoDic[fileInfo[1]] = -1 hostInfoDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), hostInfoDic.keys()) # The cloning loop. Loop over the list of files to clone and generate # a report with the result. if (emailNotif): cloneStatusDbmName = tmpFilePat + "_CLONE_STATUS_DB" cloneStatusDbm = ngamsDbm.ngamsDbm(cloneStatusDbmName, cleanUpOnDestr=0, writePerm=1) successCloneCount = 0 failedCloneCount = 0 abortCloneLoop = 0 timeAccu = 0.0 key = 0 while (1): clone_start = time.time() if (not cloneListDbm.hasKey(str(key))): break fileInfo = cloneListDbm.get(str(key)) key += 1 # Check if we have permission to run. Otherwise, stop. if (not srvObj.getThreadRunPermission()): break fio = fileInfo[0] mtPt = fileInfo[2] if (emailNotif): tmpFileList = ngamsFileList.\ ngamsFileList("FILE_CLONE_STATUS", "File: " + fio.getFileId() + "/" +\ fio.getDiskId() + "/" +\ str(fio.getFileVersion())) hostId = fileInfo[1] text = "Cloning file - File ID: %s/%d, on disk " +\ "with ID: %s on host: %s" logger.debug(text, fio.getFileId(), fio.getFileVersion(), fio.getDiskId(), hostId) # We generate a local Staging File and archive this. stagingFilename = "" try: # Check if file is marked as bad. if (fio.getFileStatus()[0] == "1"): errMsg = "File marked as bad - skipping!" raise Exception, errMsg if (targetDiskId == ""): # Try to find a disk not hosting already a file with that # ID + version. diskExemptList = [fio.getDiskId()] while (1): trgDiskInfo = ngamsDiskUtils.\ findTargetDisk(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), fio.getFormat(), 1, diskExemptList) # Check if a file with that ID + version is already # stored on the selected Target Disk. if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fio.getFileId(), fio.getFileVersion())): # This file is already stored on the given disk. # Add to the exempt list. diskExemptList.append(trgDiskInfo.getDiskId()) else: # OK, this disk should be OK, stop looking for a # suitable Target Disk. break else: try: trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\ read(srvObj.getDb(), targetDiskId) slotId = trgDiskInfo.getSlotId() storageSetId = srvObj.getCfg().\ getStorageSetFromSlotId(slotId).\ getStorageSetId() trgDiskInfo.setStorageSetId(storageSetId) except Exception: abortCloneLoop = 1 raise # We don't accept to clone onto the same disk (this would mean # overwriting). if (trgDiskInfo.getDiskId() == fio.getDiskId()): err = "Source and target files are identical" msg = "Failed in cloning file with ID: " + fio.getFileId() +\ "/Version: " + str(fio.getFileVersion()) +\ " on disk with ID: " + fio.getDiskId() +\ " on host: " + hostId + ". Reason: " + err logger.warning(msg) if (emailNotif): tmpFileList.setStatus(NGAMS_FAILURE + ": " + err) tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE")) cloneStatusDbm.addIncKey(tmpFileList) failedCloneCount += 1 continue tmpReqPropsObj = ngamsReqProps.ngamsReqProps() tmpReqPropsObj.setMimeType(fio.getFormat()) stagingFilename = ngamsHighLevelLib.\ genStagingFilename(srvObj.getCfg(), tmpReqPropsObj, trgDiskInfo, fio.getFileId()) # Receive the data into the Staging File using the urllib. if (srvObj.getHostId() != hostId): # Example: http://host:7777/RETRIEVE?file_id=id&file_version=1 ipAddress = hostInfoDic[hostId].getIpAddress() portNo = hostInfoDic[hostId].getSrvPort() fileUrl = "http://" + ipAddress + ":" + str(portNo) +\ "/RETRIEVE?" + "file_id=" + fio.getFileId() +\ "&file_version=" + str(fio.getFileVersion()) # If a specific Disk ID for the source file is given, append # this. if (fio.getDiskId()): fileUrl += "&disk_id=%s" % fio.getDiskId() # Check if host is suspended, if yes, wake it up. if (srvObj.getDb().getSrvSuspended(hostId)): logger.debug("Clone Request - Waking up suspended " +\ "NGAS Host: %s", hostId) ngamsSrvUtils.wakeUpHost(srvObj, hostId) else: fileUrl = "file:" + mtPt + "/" + fio.getFilename() logger.debug( "Receiving file via URI: %s into staging filename: %s", fileUrl, stagingFilename) # We try up to 5 times to retrieve the file in case a problem is # encountered during cloning. for attempt in range(5): try: filename, headers = urllib.urlretrieve( fileUrl, stagingFilename) _checkFile(srvObj, fio, stagingFilename, headers, checkChecksum) # If we get to this point the transfer was (probably) OK. break except Exception, e: rmFile(stagingFilename) errMsg = "Problem occurred while cloning file "+\ "via URL: " + fileUrl + " - Error: " + str(e) if (attempt < 4): errMsg += " - Retrying in 5s ..." logger.error(errMsg) time.sleep(5) else: raise Exception, errMsg # We simply copy the file into the same destination as the # source file (but on another disk). targPathName = os.path.dirname(fio.getFilename()) targFilename = os.path.basename(fio.getFilename()) complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\ "/" + targPathName) checkCreatePath(complTargPath) complFilename = os.path.normpath(complTargPath + "/" + targFilename) mvTime = mvFile(stagingFilename, complFilename) ngamsLib.makeFileReadOnly(complFilename) # Update status for new file in the DB. newFileInfo = fio.clone().setDiskId(trgDiskInfo.getDiskId()).\ setCreationDate(getFileCreationTime(complFilename)) fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fio.getFileId(), fio.getFileVersion()) newFileInfo.write(srvObj.getHostId(), srvObj.getDb()) # Update status for the Target Disk in DB + check if the disk is # completed. if (fileExists): mvTime = 0 dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\ setDiskId(trgDiskInfo.getDiskId()).\ setFileExists(fileExists).\ setFileSize(fio.getFileSize()).setIoTime(mvTime) ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj) ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId()) # Update the clone file status list. if (emailNotif): tmpFileList.setStatus(NGAMS_SUCCESS) tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE")) tmpFileList.addFileInfoObj(newFileInfo.setTag("TARGET_FILE")) cloneStatusDbm.addIncKey(tmpFileList) successCloneCount += 1 # If running as a cache archive, update the Cache New Files DBM # with the information about the new file. if (srvObj.getCachingActive()): diskId = trgDiskInfo.getDiskId() fileId = fio.getFileId() fileVer = fio.getFileVersion() filename = fio.getFilename() ngamsCacheControlThread.addEntryNewFilesDbm( srvObj, diskId, fileId, fileVer, filename) # Generate a confirmation log entry. cloneTime = time.time() - clone_start timeAccu += cloneTime msg = genLog("NGAMS_INFO_FILE_CLONED", [ fio.getFileId(), fio.getFileVersion(), fio.getDiskId(), hostId ]) msg = msg + ". Time: %.3fs. Total time: %.3fs." %\ (cloneTime, timeAccu) logger.info(msg, extra={'to_syslog': True}) except Exception, e: cloneTime = time.time() - clone_start timeAccu += cloneTime errMsg = genLog("NGAMS_ER_FILE_CLONE_FAILED", [ fio.getFileId(), fio.getFileVersion(), fio.getDiskId(), hostId, str(e) ]) if (abortCloneLoop): logger.error(errMsg, extra={'to_syslog': True}) thread.exit() else: logger.warning(errMsg) if (emailNotif): tmpFileList.setStatus(NGAMS_FAILURE + ": Error: " + errMsg) tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE")) cloneStatusDbm.addIncKey(tmpFileList) failedCloneCount += 1 # Delete Staging File if already created. if ((stagingFilename != "") and (os.path.exists(stagingFilename))): rmFile(stagingFilename)
def _cloneExec(srvObj, cloneListDbmName, tmpFilePat, targetDiskId, reqPropsObj): """ See documentation of ngamsCloneCmd._cloneThread(). This function is merely implemented in order to encapsulate the whole process to be able to clean up properly when the processing is terminated. """ cloneStatusDbm = None emailNotif = 0 checkChecksum = 1 if (reqPropsObj): if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1 if (reqPropsObj.hasHttpPar("check")): checkChecksum = int(reqPropsObj.getHttpPar("check")) # Open clone list DB. cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName) # We have to get the port numbers of the hosts where the files to be # cloned are stored. hostInfoDic = {} cloneListDbm.initKeyPtr() while (1): key, fileInfo = cloneListDbm.getNext() if (not key): break hostInfoDic[fileInfo[1]] = -1 hostInfoDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), hostInfoDic.keys()) # The cloning loop. Loop over the list of files to clone and generate # a report with the result. if (emailNotif): cloneStatusDbmName = tmpFilePat + "_CLONE_STATUS_DB" cloneStatusDbm = ngamsDbm.ngamsDbm(cloneStatusDbmName, cleanUpOnDestr = 0, writePerm = 1) successCloneCount = 0 failedCloneCount = 0 abortCloneLoop = 0 timeAccu = 0.0 key = 0 while (1): clone_start = time.time() if (not cloneListDbm.hasKey(str(key))): break fileInfo = cloneListDbm.get(str(key)) key += 1 # Check if we have permission to run. Otherwise, stop. if (not srvObj.run_async_commands): break fio = fileInfo[0] mtPt = fileInfo[2] if (emailNotif): tmpFileList = ngamsFileList.\ ngamsFileList("FILE_CLONE_STATUS", "File: " + fio.getFileId() + "/" +\ fio.getDiskId() + "/" +\ str(fio.getFileVersion())) hostId = fileInfo[1] text = "Cloning file - File ID: %s/%d, on disk " +\ "with ID: %s on host: %s" logger.debug(text, fio.getFileId(), fio.getFileVersion(), fio.getDiskId(), hostId) # We generate a local Staging File and archive this. stagingFilename = "" try: # Check if file is marked as bad. if (fio.getFileStatus()[0] == "1"): errMsg = "File marked as bad - skipping!" raise Exception(errMsg) if (targetDiskId == ""): # Try to find a disk not hosting already a file with that # ID + version. diskExemptList = [fio.getDiskId()] while (1): trgDiskInfo = ngamsDiskUtils.\ findTargetDisk(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), fio.getFormat(), 1, diskExemptList) # Check if a file with that ID + version is already # stored on the selected Target Disk. if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fio.getFileId(), fio.getFileVersion())): # This file is already stored on the given disk. # Add to the exempt list. diskExemptList.append(trgDiskInfo.getDiskId()) else: # OK, this disk should be OK, stop looking for a # suitable Target Disk. break else: try: trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\ read(srvObj.getDb(), targetDiskId) slotId = trgDiskInfo.getSlotId() storageSetId = srvObj.getCfg().\ getStorageSetFromSlotId(slotId).\ getStorageSetId() trgDiskInfo.setStorageSetId(storageSetId) except Exception: abortCloneLoop = 1 raise # We don't accept to clone onto the same disk (this would mean # overwriting). if (trgDiskInfo.getDiskId() == fio.getDiskId()): err = "Source and target files are identical" msg = "Failed in cloning file with ID: " + fio.getFileId() +\ "/Version: " + str(fio.getFileVersion()) +\ " on disk with ID: " + fio.getDiskId() +\ " on host: " + hostId + ". Reason: " + err logger.warning(msg) if (emailNotif): tmpFileList.setStatus(NGAMS_FAILURE + ": " + err) tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE")) cloneStatusDbm.addIncKey(tmpFileList) failedCloneCount += 1 continue tmpReqPropsObj = ngamsReqProps.ngamsReqProps() tmpReqPropsObj.setMimeType(fio.getFormat()) stagingFilename = ngamsHighLevelLib.\ genStagingFilename(srvObj.getCfg(), tmpReqPropsObj, trgDiskInfo, fio.getFileId()) # Receive the data into the Staging File using the urllib. if (srvObj.getHostId() != hostId): # Example: http://host:7777/RETRIEVE?file_id=id&file_version=1 ipAddress = hostInfoDic[hostId].getIpAddress() portNo = hostInfoDic[hostId].getSrvPort() fileUrl = "http://" + ipAddress + ":" + str(portNo) +\ "/RETRIEVE?" + "file_id=" + fio.getFileId() +\ "&file_version=" + str(fio.getFileVersion()) # If a specific Disk ID for the source file is given, append # this. if (fio.getDiskId()): fileUrl += "&disk_id=%s" % fio.getDiskId() # Check if host is suspended, if yes, wake it up. if (srvObj.getDb().getSrvSuspended(hostId)): logger.debug("Clone Request - Waking up suspended " +\ "NGAS Host: %s", hostId) ngamsSrvUtils.wakeUpHost(srvObj, hostId) else: fileUrl = "file:" + mtPt + "/" + fio.getFilename() logger.debug("Receiving file via URI: %s into staging filename: %s", fileUrl, stagingFilename) # We try up to 5 times to retrieve the file in case a problem is # encountered during cloning. for attempt in range(5): try: filename, headers = urlrequest.urlretrieve(fileUrl, stagingFilename) _checkFile(srvObj, fio, stagingFilename, headers, checkChecksum) # If we get to this point the transfer was (probably) OK. break except Exception as e: rmFile(stagingFilename) errMsg = "Problem occurred while cloning file "+\ "via URL: " + fileUrl + " - Error: " + str(e) if (attempt < 4): errMsg += " - Retrying in 5s ..." logger.error(errMsg) time.sleep(5) else: raise Exception(errMsg) # We simply copy the file into the same destination as the # source file (but on another disk). targPathName = os.path.dirname(fio.getFilename()) targFilename = os.path.basename(fio.getFilename()) complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\ "/" + targPathName) checkCreatePath(complTargPath) complFilename = os.path.normpath(complTargPath + "/"+targFilename) mvTime = mvFile(stagingFilename, complFilename) ngamsLib.makeFileReadOnly(complFilename) # Update status for new file in the DB. newFileInfo = fio.clone().setDiskId(trgDiskInfo.getDiskId()).\ setCreationDate(getFileCreationTime(complFilename)) fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fio.getFileId(), fio.getFileVersion()) newFileInfo.write(srvObj.getHostId(), srvObj.getDb()) # Update status for the Target Disk in DB + check if the disk is # completed. if (fileExists): mvTime = 0 dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\ setDiskId(trgDiskInfo.getDiskId()).\ setFileExists(fileExists).\ setFileSize(fio.getFileSize()).setIoTime(mvTime) ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj) ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId()) # Update the clone file status list. if (emailNotif): tmpFileList.setStatus(NGAMS_SUCCESS) tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE")) tmpFileList.addFileInfoObj(newFileInfo.setTag("TARGET_FILE")) cloneStatusDbm.addIncKey(tmpFileList) successCloneCount += 1 # If running as a cache archive, update the Cache New Files DBM # with the information about the new file. if (srvObj.getCachingActive()): diskId = trgDiskInfo.getDiskId() fileId = fio.getFileId() fileVer = fio.getFileVersion() filename = fio.getFilename() ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId, fileId, fileVer, filename) # Generate a confirmation log entry. cloneTime = time.time() - clone_start timeAccu += cloneTime msg = genLog("NGAMS_INFO_FILE_CLONED", [fio.getFileId(), fio.getFileVersion(), fio.getDiskId(), hostId]) msg = msg + ". Time: %.3fs. Total time: %.3fs." %\ (cloneTime, timeAccu) logger.info(msg, extra={'to_syslog': True}) except Exception as e: cloneTime = time.time() - clone_start timeAccu += cloneTime errMsg = genLog("NGAMS_ER_FILE_CLONE_FAILED", [fio.getFileId(), fio.getFileVersion(), fio.getDiskId(), hostId, str(e)]) if (abortCloneLoop): logger.error(errMsg, extra={'to_syslog': True}) return else: logger.warning(errMsg) if (emailNotif): tmpFileList.setStatus(NGAMS_FAILURE + ": Error: " + errMsg) tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE")) cloneStatusDbm.addIncKey(tmpFileList) failedCloneCount += 1 # Delete Staging File if already created. if ((stagingFilename != "") and (os.path.exists(stagingFilename))): rmFile(stagingFilename) # Calculate time statistics. if (reqPropsObj): ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj.\ incActualCount(1), timeAccu) # Final update of the Request Status. if (reqPropsObj): complPercent = (100.0 * (float(reqPropsObj.getActualCount()) / float(reqPropsObj.getExpectedCount()))) reqPropsObj.setCompletionPercent(complPercent, 1) reqPropsObj.setCompletionTime(1) srvObj.updateRequestDb(reqPropsObj) # Send Clone Report with list of files cloned to a possible # requestor(select) of this. totFiles = (successCloneCount + failedCloneCount) if (emailNotif): xmlStat = 0 # TODO: Generation of XML status report is disabled since we cannot # handle for the moment XML documents with 1000s of elements. if (xmlStat): cloneStatusFileList = ngamsFileList.\ ngamsFileList("FILE_CLONING_STATUS_REPORT", "File Cloning Status Report") fileCount = 0 while (fileCount < cloneStatusDbm.getCount()): tmpFileList = cloneStatusDbm.get(str(fileCount)) cloneStatusFileList.addFileListObj(tmpFileList) # Make overall status. cloneStatusFileList.setStatus("SUCCESS: " +\ str(successCloneCount) +\ ", FAILURE: " +\ str(failedCloneCount) +\ ", NOT DONE: " +\ str(len(cloneStatusFileList) -\ successCloneCount -\ failedCloneCount)) status = srvObj.genStatus(NGAMS_SUCCESS, "CLONE command status report").\ addFileList(cloneStatusFileList) statRep = status.genXmlDoc(0, 0, 0, 1, 0) statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep) mimeType = NGAMS_XML_MT else: # Generate a 'simple' ASCII report. statRep = tmpFilePat + "_NOTIF_EMAIL.txt" fo = open(statRep, "w") if (reqPropsObj.hasHttpPar("disk_id")): diskId = reqPropsObj.getHttpPar("disk_id") else: diskId = "-----" if (reqPropsObj.hasHttpPar("file_id")): fileId = reqPropsObj.getHttpPar("file_id") else: fileId = "-----" if (reqPropsObj.hasHttpPar("file_version")): fileVersion = reqPropsObj.getHttpPar("file_version") else: fileVersion = "-----" tmpFormat = "CLONE STATUS REPORT:\n\n" +\ "==Summary:\n\n" +\ "Date: %s\n" +\ "NGAS Host: %s\n" +\ "Disk ID: %s\n" +\ "File ID: %s\n" +\ "File Version: %s\n" +\ "Total Number of Files: %d\n" +\ "Number of Cloned Files: %d\n" +\ "Number of Failed Files: %d\n" +\ "Total processing time (s): %.3f\n" +\ "Handling time per file (s): %.3f\n\n" +\ "==File List:\n\n" fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), diskId, fileId, str(fileVersion), totFiles, successCloneCount, failedCloneCount, timeAccu, (timeAccu / totFiles))) tmpFormat = "%-70s %-70s %-7s\n" fo.write(tmpFormat % ("Source File", "Target File", "Status")) fo.write(tmpFormat % (70 * "-", 70 * "-", 7 * "-")) key = 1 while (1): if (not cloneStatusDbm.hasKey(str(key))): break tmpFileList = cloneStatusDbm.get(str(key)) key += 1 srcFileObj = tmpFileList.getFileInfoObjList()[0] srcFile = "%s/%s/%d" % (srcFileObj.getDiskId(), srcFileObj.getFileId(), srcFileObj.getFileVersion()) if (tmpFileList.getStatus() == NGAMS_SUCCESS): trgFileObj = tmpFileList.getFileInfoObjList()[1] trgFile = "%s/%s/%d" % (trgFileObj.getDiskId(), trgFileObj.getFileId(), trgFileObj.getFileVersion()) else: trgFile = "-----" fo.write(tmpFormat % (srcFile,trgFile,tmpFileList.getStatus())) fo.write(149 * "-") fo.write("\n\n==END\n") fo.close() mimeType = NGAMS_TEXT_MT # Send out the status report. emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",") attachmentName = "CloneStatusReport" if (reqPropsObj.hasHttpPar("disk_id")): attachmentName += "-" + reqPropsObj.getHttpPar("disk_id") if (reqPropsObj.hasHttpPar("file_id")): attachmentName += "-" + reqPropsObj.getHttpPar("file_id") if (reqPropsObj.hasHttpPar("file_version")): attachmentName += "-" + reqPropsObj.getHttpPar("file_version") ngamsNotification.notify(srvObj.host_id, srvObj.cfg, NGAMS_NOTIF_INFO, "CLONE STATUS REPORT", statRep, recList=emailAdrList, force=1, contentType=mimeType, attachmentName=attachmentName) del cloneStatusDbm rmFile(cloneStatusDbmName + "*") rmFile(statRep) if (cloneListDbm): del cloneListDbm rmFile(cloneListDbmName + "*") logger.info("_cloneExec(). Total time: %.3fs. Average time per file: %.3fs.", timeAccu, (timeAccu / totFiles))
def _registerExec(srvObj, fileListDbmName, tmpFilePat, diskInfoDic, reqPropsObj=None): """ Register the files listed in the File List DBM (ngamsDbm), which match the mime-type(s) either specified in the 'mimeType' parameter, or if this is not specified, which match all the mime-types specified in the configuration file. When the registration procedure has been executed, the function sends an Email Notification message indicating which files were registered if the HTTP parameter 'notif_email' is given. The functions creates a File Info Objects per file handled and writes this in a temporary file, which is a DBM file. The keys in this DB is simply the file number in the sequence of files handled, pointing to a pickled ngamsFileInfo object. Each of the File Info Objects indicates if the file was registered or not. This is done by setting the tag of the File Info Object to one of the following values: REGISTERED: The file was successfully registered in the NGAS DB FAILED: The file was selected for registration but could not be properly registered because of inconsistencies. The status will be of the format: 'FAILED[: <reason>]'. REJECTED: A file found under the specified path directory was not accepted for cloning, usually because the mime-type was not correct. The status will be of the format: 'REJECTED[: <reason>]'. Note, that registration is comparable to archiving of files. For that reason a DAPI must be provided for each type of file that should be registered. If this is not fullfilled, the file registration will fail. Only files stored on one of the NGAS disks configured in the configuration file, are considered. Files stored in other locations are ignored. srvObj: Instance of NG/AMS Server object (ngamsServer). fileListDbmName: Name of a DBM containing the information about the files to be registered. Each element in the list is referred to by a key, which is a number. These points to a pickled list for each file containing the following information: [<Filename>, <Disk ID>, <Mime-Type>] The information for each disk concerned is also contained in the DB referred to by its Disk ID and by its mount point. The data is a pickled instance of the ngamsDiskInfo class (string). tmpFilePat: Pattern for temporary files used during the registration process (string). diskInfoDic: Dictionary with Disk IDs as keys pointing to the info about the disk (dictionary/ngamsDiskInfo). reqPropsObj: If an NG/AMS Request Properties Object is given, the Request Status will be updated as the request is carried out (ngamsReqProps). Returns: Void. """ emailNotif = 0 if (reqPropsObj): if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1 # Create the temporary BSD DB to contain the information for the # Email Notification Message. if (emailNotif): regDbmName = tmpFilePat + "_NOTIF_EMAIL" regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm=1) # Open the DBM containing the list of files to (possibly) register. fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1) # Want to parse files in alphabetical order. # TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting. tmpFileList = tmpFilePat + "_FILE_LIST" rmFile(tmpFileList) with open(tmpFileList, "wb") as fo: fileListDbm.initKeyPtr() while (1): dbmKey, fileInfo = fileListDbm.getNext() if (not dbmKey): break fo.write(dbmKey + b"\n") sortFileList = tmpFilePat + "_SORT_FILE_LIST" rmFile(sortFileList) shellCmd = "sort %s > %s" % (tmpFileList, sortFileList) stat, out, err = ngamsCore.execCmd(shellCmd) if (stat != 0): raise Exception("Error executing command: %s. Error: %s, %s" %\ (shellCmd, str(out), str(err))) rmFile(tmpFileList) # Go through each file in the list, check if the mime-type is among the # ones, which apply for registration. If yes try to register the file # by invoking the corresponding DAPI on the file. fileRegCount = 0 fileFailCount = 0 fileRejectCount = 0 regTimeAccu = 0.0 fileCount = 0 fo = open(sortFileList) run = 1 while (run): reg_start = time.time() dbmKey = fo.readline() if (dbmKey.strip() == ""): run = 0 continue fileInfo = fileListDbm.get(dbmKey[0:-1]) filename = fileInfo[0] diskId = fileInfo[1] mimeType = fileInfo[2] # Register the file. Check first, that exactly this file is # not already registered. In case it is, the file will be rejected. regPi = srvObj.getCfg().register_plugins[mimeType] logger.debug("Plugin found for %s: %s", mimeType, regPi) params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars) tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\ setMimeType(mimeType).\ setStagingFilename(filename).\ setTargDiskInfo(diskInfoDic[diskId]).\ setHttpMethod(NGAMS_HTTP_GET).\ setCmd(NGAMS_REGISTER_CMD).\ setSize(os.path.getsize(filename)).\ setFileUri(filename).\ setNoReplication(1) tmpFileObj = ngamsFileInfo.ngamsFileInfo() try: # Invoke Registration Plug-In. piName = regPi.name plugInMethod = loadPlugInEntryPoint(piName) piRes = plugInMethod(srvObj, tmpReqPropsObj, params) del tmpReqPropsObj # Check if this file is already registered on this disk. In case # yes, it is not registered again. files = srvObj.db.getFileSummary1(srvObj.getHostId(), [piRes.getDiskId()], [piRes.getFileId()]) fileRegistered = 0 for tmpFileInfo in files: tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT] tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME] tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\ tmpFilename) if (tmpComplFilename == filename): fileRegistered = 1 break if (fileRegistered): fileRejectCount += 1 tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\ "and path: %s is already registered on disk " +\ "with Disk ID: %s" tmpMsg = tmpMsgForm % (piRes.getFileId(), piRes.getFileVersion(), filename, piRes.getDiskId()) logger.warning(tmpMsg + ". File is not registered again.") if (emailNotif): tmpFileObj.\ setDiskId(diskId).setFilename(filename).\ setTag(tmpMsg) regDbm.addIncKey(tmpFileObj) if (reqPropsObj): reqPropsObj.incActualCount(1) ngamsHighLevelLib.stdReqTimeStatUpdate( srvObj, reqPropsObj, regTimeAccu) regTimeAccu += time.time() - reg_start fileCount += 1 continue # Calculate checksum. We maintain the old name for backwards # compatibility crc_variant = srvObj.cfg.getCRCVariant() if crc_variant == ngamsFileUtils.CHECKSUM_CRC32_INCONSISTENT: crc_variant = 'ngamsGenCrc32' checksum = ngamsFileUtils.get_checksum(65536, filename, crc_variant) or '' # Move file and update information about file in the NGAS DB. mvFile(filename, piRes.getCompleteFilename()) ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum, crc_variant) ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes) ngamsLib.makeFileReadOnly(piRes.getCompleteFilename()) if (emailNotif): uncomprSize = piRes.getUncomprSize() ingestDate = time.time() creDateSecs = getFileCreationTime(filename) tmpFileObj.\ setDiskId(diskId).\ setFilename(filename).\ setFileId(piRes.getFileId()).\ setFileVersion(piRes.getFileVersion()).\ setFormat(piRes.getFormat()).\ setFileSize(piRes.getFileSize()).\ setUncompressedFileSize(uncomprSize).\ setCompression(piRes.getCompression()).\ setIngestionDate(ingestDate).\ setIgnore(0).\ setChecksum(checksum).\ setChecksumPlugIn(crc_variant).\ setFileStatus(NGAMS_FILE_STATUS_OK).\ setCreationDate(creDateSecs).\ setTag("REGISTERED") fileRegCount += 1 # If running as a cache archive, update the Cache New Files DBM # with the information about the new file. if (srvObj.getCachingActive()): fileVer = fio.getFileVersion() ngamsCacheControlThread.addEntryNewFilesDbm( srvObj, diskId, piRes.getFileId(), fileVer, filename) # Generate a confirmation log entry. msg = genLog("NGAMS_INFO_FILE_REGISTERED", [ filename, piRes.getFileId(), piRes.getFileVersion(), piRes.getFormat() ]) time.sleep(0.005) regTime = time.time() - reg_start msg = msg + ". Time: %.3fs." % (regTime) logger.info(msg, extra={'to_syslog': 1}) except Exception as e: errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)]) logger.error(errMsg) if (emailNotif): tmpFileObj.\ setDiskId(diskId).setFilename(filename).\ setTag(errMsg) fileFailCount += 1 regTime = time.time() - reg_start # TODO (rtobar, 2016-01): Why don't we raise an exception here? # Otherwise the command appears as successful on the # client-side # Add the file information in the registration report. if (emailNotif): regDbm.addIncKey(tmpFileObj) # Update request status time information. regTimeAccu += regTime if (reqPropsObj): reqPropsObj.incActualCount(1) ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj, regTimeAccu) fileCount += 1 fo.close() rmFile(sortFileList) if (emailNotif): regDbm.sync() del fileListDbm rmFile(fileListDbmName + "*") # Final update of the Request Status. if (reqPropsObj): if (reqPropsObj.getExpectedCount() and reqPropsObj.getActualCount()): complPercent = (100.0 * (float(reqPropsObj.getActualCount()) / float(reqPropsObj.getExpectedCount()))) else: complPercent = 100.0 reqPropsObj.setCompletionPercent(complPercent, 1) reqPropsObj.setCompletionTime(1) srvObj.updateRequestDb(reqPropsObj) # Send Register Report with list of files cloned to a possible # requestor(select) of this. if (emailNotif): xmlStat = 0 if (xmlStat): # Create an instance of the File List Class, used to store the # Registration Report. regStat = ngamsFileList.\ ngamsFileList("FILE_REGISTRATION_STATUS", "Status report for file " +\ "registration") # Loop over the file objects in the BSD DB and add these # in the status object. regDbm.initKeyPtr() while (1): key, tmpFileObj = regDbm.getNext() if (not key): break regStat.addFileInfoObj(tmpFileObj) # Set overall status of registration procedure. regStat.setStatus("Files Found: " + str(fileCount + 1) + ", "+\ "Files Registered: " + str(fileRegCount) +\ ", " +\ "Files Failed: " + str(fileFailCount) + ", " +\ "Files Rejected: " + str(fileRejectCount)) status = srvObj.genStatus(NGAMS_SUCCESS, "REGISTER command status report").\ addFileList(regStat) statRep = status.genXmlDoc() statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep) mimeType = NGAMS_XML_MT else: # Generate a 'simple' ASCII report. statRep = tmpFilePat + "_NOTIF_EMAIL.txt" fo = open(statRep, "w") if (reqPropsObj): path = reqPropsObj.getHttpPar("path") else: path = "-----" if (fileCount): timePerFile = (regTimeAccu / fileCount) else: timePerFile = 0 tmpFormat = "REGISTER STATUS REPORT:\n\n" +\ "==Summary:\n\n" +\ "Date: %s\n" +\ "NGAS Host: %s\n" +\ "Search Path: %s\n" +\ "Total Number of Files: %d\n" +\ "Number of Registered Files: %d\n" +\ "Number of Failed Files: %d\n" +\ "Number of Rejected Files: %d\n" +\ "Total processing time (s): %.3f\n" +\ "Handling time per file (s): %.3f\n\n" +\ "==File List:\n\n" fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), path, fileCount, fileRegCount, fileFailCount, fileRejectCount, regTimeAccu, timePerFile)) tmpFormat = "%-80s %-32s %-3s %-10s\n" fo.write(tmpFormat %\ ("Filename", "File ID", "Ver", "Status")) fo.write(tmpFormat % (80 * "-", 32 * "-", 3 * "-", 10 * "-")) regDbm.initKeyPtr() while (1): key, tmpFileObj = regDbm.getNext() if (not key): break mtPt = diskInfoDic[tmpFileObj.getDiskId()].getMountPoint() filename = os.path.normpath(mtPt + "/" +\ tmpFileObj.getFilename()) line = tmpFormat %\ (filename, tmpFileObj.getFileId(), str(tmpFileObj.getFileVersion()), tmpFileObj.getTag()) fo.write(line) fo.write(128 * "-") fo.write("\n\n==END\n") fo.close() mimeType = NGAMS_TEXT_MT # Send out the status report. emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",") attachmentName = "RegisterStatusReport" if (reqPropsObj.hasHttpPar("path")): attachmentName += "-" + reqPropsObj.getHttpPar("path").\ replace("/", "_") ngamsNotification.notify(srvObj.host_id, srvObj.cfg, NGAMS_NOTIF_INFO, "REGISTER STATUS REPORT", statRep, recList=emailAdrList, force=1, contentType=mimeType, attachmentName=attachmentName) del regDbm rmFile(regDbmName + "*") rmFile(statRep) # Generate final status log + exit. if (fileCount > 0): timePerFile = (regTimeAccu / fileCount) else: timePerFile = 0.0 msg = "Registration procedure finished processing Register Request - " + \ "terminating. Files handled: %d. Total time: %.3fs. " + \ "Average time per file: %.3fs." logger.debug(msg, fileCount, regTimeAccu, timePerFile)
def process_mirroring_tasks(mirroring_tasks_queue, target_node, ith_thread, n_tasks, srvObj): """ Process mirroring tasks described in the input mirroring_tasks list INPUT: mirroring_tasks_queue Queue of the mirroring tasks assigned to the input server target_node string, Full qualified name of the target node ith_thread int, Thread number n_tasks int, Initial size of the queue srvObj ngamsServer, Reference to NG/AMS server class object RETURNS: Void """ logger.debug("Inside mirror worker worker %d to mirror files to %s", ith_thread, target_node) try: # Loop on the mirroring_tasks_queue while 1: # Get tasks from queue if (len(mirroring_tasks_queue)): try: if (ith_thread % 2): item = mirroring_tasks_queue.pop() else: item = mirroring_tasks_queue.popleft() except: break else: break # get all the fields we're interested in and form them into a command to fetch the files. # these fields were extracted in get_list_mirroring_tasks() logger.debug("next task: %r", item) staging_file = str(item[1]) mimeType = str(item[3]) checksum = str(item[4]) rowid = str(item[2]) file_id = str(item[9]) fileSize = str(item[0]) fileInfo = {} fileInfo['sourceHost'] = str(item[5]) fileInfo['diskId'] = str(item[6]) fileInfo['hostId'] = str(item[7]) fileInfo['fileVersion'] = str(item[8]) fileInfo['fileId'] = file_id logger.info( "Processing mirroring task (Target node: %s, file size: %s, Thread: %d) file info: %s", target_node, fileSize, ith_thread, str(fileInfo)) # Initialize ngamsReqProps object by just specifing the fileURI and the mime type reqPropObj = ngamsReqProps.ngamsReqProps() reqPropObj.setMimeType(mimeType) reqPropObj.checksum = checksum reqPropObj.fileinfo = fileInfo reqPropObj.setSize(fileSize) # Start clock start = time.time() (stgFilename, targetDiskInfo) = calculateStagingName(srvObj, file_id, staging_file) reqPropObj.setStagingFilename(stgFilename) reqPropObj.setTargDiskInfo(targetDiskInfo) try: # Construct query to update ingestion date, ingestion time and status query = "update ngas_mirroring_bookkeeping set status='FETCHING', " query += "staging_file = {0}, " query += "attempt = nvl(attempt + 1, 1) " query += "where rowid = {1}" # Add query to the queue srvObj.getDb().query2(query, args=(reqPropObj.getStagingFilename(), rowid)) logger.info("Mirroring file: %s", file_id) ngamsCmd_MIRRARCHIVE.handleCmd(srvObj, reqPropObj) status = "SUCCESS" except ngamsFailedDownloadException.FailedDownloadException: # Something bad happened... logger.exception("Failed to fetch %s", file_id) status = "FAILURE" except ngamsFailedDownloadException.AbortedException: logger.warning("File fetch aborted: %s", file_id) status = "ABORTED" except ngamsFailedDownloadException.PostponeException: logger.exception( "Failed to fetch %s - will try to resume on next iteration", file_id) status = "TORESUME" except Exception: # this clause should never be reached logger.exception("Fetch failed in an unexpected way") status = "FAILURE" # Get time elapsed elapsed_time = (time.time() - start) # Construct query to update ingestion date, ingestion time and status query = "update ngas_mirroring_bookkeeping set status = {0}," if (status != 'TORESUME'): query += "staging_file = null, " query += "ingestion_date = {1}," query += "ingestion_time = nvl(ingestion_time, 0.0) + {2} " query += "where rowid = {3}" args = (status, toiso8601(fmt=FMT_DATETIME_NOMSEC) + ":000", elapsed_time, rowid) srvObj.getDb().query2(query, args=args) # Log message for mirroring task processed completion = 100 * (n_tasks - len(mirroring_tasks_queue)) / float(n_tasks) logger.info( "Mirroring task (Target node: %s Thread: %d) processed in %fs (%s), completion: %f%%: %s", target_node, ith_thread, elapsed_time, status, completion, str(fileInfo)) logger.info("Mirroring Worker complete") except Exception: logger.exception("Error while running mirroring worker") # Return Void return
def _registerExec(srvObj, fileListDbmName, tmpFilePat, diskInfoDic, reqPropsObj = None): """ Register the files listed in the File List DBM (ngamsDbm), which match the mime-type(s) either specified in the 'mimeType' parameter, or if this is not specified, which match all the mime-types specified in the configuration file. When the registration procedure has been executed, the function sends an Email Notification message indicating which files were registered if the HTTP parameter 'notif_email' is given. The functions creates a File Info Objects per file handled and writes this in a temporary file, which is a DBM file. The keys in this DB is simply the file number in the sequence of files handled, pointing to a pickled ngamsFileInfo object. Each of the File Info Objects indicates if the file was registered or not. This is done by setting the tag of the File Info Object to one of the following values: REGISTERED: The file was successfully registered in the NGAS DB FAILED: The file was selected for registration but could not be properly registered because of inconsistencies. The status will be of the format: 'FAILED[: <reason>]'. REJECTED: A file found under the specified path directory was not accepted for cloning, usually because the mime-type was not correct. The status will be of the format: 'REJECTED[: <reason>]'. Note, that registration is comparable to archiving of files. For that reason a DAPI must be provided for each type of file that should be registered. If this is not fullfilled, the file registration will fail. Only files stored on one of the NGAS disks configured in the configuration file, are considered. Files stored in other locations are ignored. srvObj: Instance of NG/AMS Server object (ngamsServer). fileListDbmName: Name of a DBM containing the information about the files to be registered. Each element in the list is referred to by a key, which is a number. These points to a pickled list for each file containing the following information: [<Filename>, <Disk ID>, <Mime-Type>] The information for each disk concerned is also contained in the DB referred to by its Disk ID and by its mount point. The data is a pickled instance of the ngamsDiskInfo class (string). tmpFilePat: Pattern for temporary files used during the registration process (string). diskInfoDic: Dictionary with Disk IDs as keys pointing to the info about the disk (dictionary/ngamsDiskInfo). reqPropsObj: If an NG/AMS Request Properties Object is given, the Request Status will be updated as the request is carried out (ngamsReqProps). Returns: Void. """ T = TRACE() emailNotif = 0 if (reqPropsObj): if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1 # Create the temporary BSD DB to contain the information for the # Email Notification Message. if (emailNotif): regDbmName = tmpFilePat + "_NOTIF_EMAIL" regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm = 1) # Open the DBM containing the list of files to (possibly) register. fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm = 1) # Want to parse files in alphabetical order. # TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting. tmpFileList = tmpFilePat + "_FILE_LIST" rmFile(tmpFileList) fo = open(tmpFileList, "w") fileListDbm.initKeyPtr() while (1): dbmKey, fileInfo = fileListDbm.getNext() if (not dbmKey): break fo.write(dbmKey + "\n") fo.close() sortFileList = tmpFilePat + "_SORT_FILE_LIST" rmFile(sortFileList) shellCmd = "sort %s > %s" % (tmpFileList, sortFileList) stat, out = commands.getstatusoutput(shellCmd) if (stat != 0): raise Exception, "Error executing command: %s. Error: %s" %\ (shellCmd, str(out)) rmFile(tmpFileList) # Go through each file in the list, check if the mime-type is among the # ones, which apply for registration. If yes try to register the file # by invoking the corresponding DAPI on the file. checksumPlugIn = srvObj.getCfg().getChecksumPlugIn().strip() fileRegCount = 0 fileFailCount = 0 fileRejectCount = 0 regTimeAccu = 0.0 fileCount = 0 fo = open(sortFileList) run = 1 while (run): reg_start = time.time() dbmKey = fo.readline() if (dbmKey.strip() == ""): run = 0 continue fileInfo = fileListDbm.get(dbmKey[0:-1]) filename = fileInfo[0] diskId = fileInfo[1] mimeType = fileInfo[2] # Register the file. Check first, that exactly this file is # not already registered. In case it is, the file will be rejected. regPi = srvObj.getCfg().register_plugins[mimeType] logger.debug("Plugin found for %s: %s", mimeType, regPi) params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars) tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\ setMimeType(mimeType).\ setStagingFilename(filename).\ setTargDiskInfo(diskInfoDic[diskId]).\ setHttpMethod(NGAMS_HTTP_GET).\ setCmd(NGAMS_REGISTER_CMD).\ setSize(os.path.getsize(filename)).\ setFileUri(filename).\ setNoReplication(1) tmpFileObj = ngamsFileInfo.ngamsFileInfo() try: # Invoke Registration Plug-In. piName = regPi.name plugInMethod = loadPlugInEntryPoint(piName) piRes = plugInMethod(srvObj, tmpReqPropsObj, params) del tmpReqPropsObj # Check if this file is already registered on this disk. In case # yes, it is not registered again. files = srvObj.db.getFileSummary1(srvObj.getHostId(), [piRes.getDiskId()], [piRes.getFileId()]) fileRegistered = 0 for tmpFileInfo in files: tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT] tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME] tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\ tmpFilename) if (tmpComplFilename == filename): fileRegistered = 1 break if (fileRegistered): fileRejectCount += 1 tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\ "and path: %s is already registered on disk " +\ "with Disk ID: %s" tmpMsg = tmpMsgForm % (piRes.getFileId(), piRes.getFileVersion(), filename, piRes.getDiskId()) logger.warning(tmpMsg + ". File is not registered again.") if (emailNotif): tmpFileObj.\ setDiskId(diskId).setFilename(filename).\ setTag(tmpMsg) regDbm.addIncKey(tmpFileObj) if (reqPropsObj): reqPropsObj.incActualCount(1) ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj, regTimeAccu) regTimeAccu += time.time() - reg_start fileCount += 1 continue # Calculate checksum (if plug-in specified). if (checksumPlugIn != ""): logger.debug("Invoking Checksum Plug-In: %s to handle file: %s", checksumPlugIn, filename) plugInMethod = loadPlugInEntryPoint(checksumPlugIn) checksum = plugInMethod(srvObj, filename, 0) else: checksum = "" # Move file and update information about file in the NGAS DB. mvFile(filename, piRes.getCompleteFilename()) ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum, checksumPlugIn) ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes) ngamsLib.makeFileReadOnly(piRes.getCompleteFilename()) if (emailNotif): uncomprSize = piRes.getUncomprSize() ingestDate = time.time() creDateSecs = getFileCreationTime(filename) tmpFileObj.\ setDiskId(diskId).\ setFilename(filename).\ setFileId(piRes.getFileId()).\ setFileVersion(piRes.getFileVersion()).\ setFormat(piRes.getFormat()).\ setFileSize(piRes.getFileSize()).\ setUncompressedFileSize(uncomprSize).\ setCompression(piRes.getCompression()).\ setIngestionDate(ingestDate).\ setIgnore(0).\ setChecksum(checksum).\ setChecksumPlugIn(checksumPlugIn).\ setFileStatus(NGAMS_FILE_STATUS_OK).\ setCreationDate(creDateSecs).\ setTag("REGISTERED") fileRegCount += 1 # If running as a cache archive, update the Cache New Files DBM # with the information about the new file. if (srvObj.getCachingActive()): fileVer = fio.getFileVersion() ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId, piRes.getFileId(), fileVer, filename) # Generate a confirmation log entry. msg = genLog("NGAMS_INFO_FILE_REGISTERED", [filename, piRes.getFileId(), piRes.getFileVersion(), piRes.getFormat()]) time.sleep(0.005) regTime = time.time() - reg_start msg = msg + ". Time: %.3fs." % (regTime) logger.info(msg, extra={'to_syslog': 1}) except Exception, e: errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)]) logger.error(errMsg) if (emailNotif): tmpFileObj.\ setDiskId(diskId).setFilename(filename).\ setTag(errMsg) fileFailCount += 1 regTime = time.time() - reg_start # TODO (rtobar, 2016-01): Why don't we raise an exception here? # Otherwise the command appears as successful on the # client-side # Add the file information in the registration report. if (emailNotif): regDbm.addIncKey(tmpFileObj) # Update request status time information. regTimeAccu += regTime if (reqPropsObj): reqPropsObj.incActualCount(1) ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj, regTimeAccu) fileCount += 1