def _cloneExec(srvObj, cloneListDbmName, tmpFilePat, targetDiskId, reqPropsObj): """ See documentation of ngamsCloneCmd._cloneThread(). This function is merely implemented in order to encapsulate the whole process to be able to clean up properly when the processing is terminated. """ cloneStatusDbm = None emailNotif = 0 checkChecksum = 1 if (reqPropsObj): if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1 if (reqPropsObj.hasHttpPar("check")): checkChecksum = int(reqPropsObj.getHttpPar("check")) # Open clone list DB. cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName) # We have to get the port numbers of the hosts where the files to be # cloned are stored. hostInfoDic = {} cloneListDbm.initKeyPtr() while (1): key, fileInfo = cloneListDbm.getNext() if (not key): break hostInfoDic[fileInfo[1]] = -1 hostInfoDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), hostInfoDic.keys()) # The cloning loop. Loop over the list of files to clone and generate # a report with the result. if (emailNotif): cloneStatusDbmName = tmpFilePat + "_CLONE_STATUS_DB" cloneStatusDbm = ngamsDbm.ngamsDbm(cloneStatusDbmName, cleanUpOnDestr = 0, writePerm = 1) successCloneCount = 0 failedCloneCount = 0 abortCloneLoop = 0 timeAccu = 0.0 key = 0 while (1): clone_start = time.time() if (not cloneListDbm.hasKey(str(key))): break fileInfo = cloneListDbm.get(str(key)) key += 1 # Check if we have permission to run. Otherwise, stop. if (not srvObj.run_async_commands): break fio = fileInfo[0] mtPt = fileInfo[2] if (emailNotif): tmpFileList = ngamsFileList.\ ngamsFileList("FILE_CLONE_STATUS", "File: " + fio.getFileId() + "/" +\ fio.getDiskId() + "/" +\ str(fio.getFileVersion())) hostId = fileInfo[1] text = "Cloning file - File ID: %s/%d, on disk " +\ "with ID: %s on host: %s" logger.debug(text, fio.getFileId(), fio.getFileVersion(), fio.getDiskId(), hostId) # We generate a local Staging File and archive this. stagingFilename = "" try: # Check if file is marked as bad. if (fio.getFileStatus()[0] == "1"): errMsg = "File marked as bad - skipping!" raise Exception(errMsg) if (targetDiskId == ""): # Try to find a disk not hosting already a file with that # ID + version. diskExemptList = [fio.getDiskId()] while (1): trgDiskInfo = ngamsDiskUtils.\ findTargetDisk(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), fio.getFormat(), 1, diskExemptList) # Check if a file with that ID + version is already # stored on the selected Target Disk. if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fio.getFileId(), fio.getFileVersion())): # This file is already stored on the given disk. # Add to the exempt list. diskExemptList.append(trgDiskInfo.getDiskId()) else: # OK, this disk should be OK, stop looking for a # suitable Target Disk. break else: try: trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\ read(srvObj.getDb(), targetDiskId) slotId = trgDiskInfo.getSlotId() storageSetId = srvObj.getCfg().\ getStorageSetFromSlotId(slotId).\ getStorageSetId() trgDiskInfo.setStorageSetId(storageSetId) except Exception: abortCloneLoop = 1 raise # We don't accept to clone onto the same disk (this would mean # overwriting). if (trgDiskInfo.getDiskId() == fio.getDiskId()): err = "Source and target files are identical" msg = "Failed in cloning file with ID: " + fio.getFileId() +\ "/Version: " + str(fio.getFileVersion()) +\ " on disk with ID: " + fio.getDiskId() +\ " on host: " + hostId + ". Reason: " + err logger.warning(msg) if (emailNotif): tmpFileList.setStatus(NGAMS_FAILURE + ": " + err) tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE")) cloneStatusDbm.addIncKey(tmpFileList) failedCloneCount += 1 continue tmpReqPropsObj = ngamsReqProps.ngamsReqProps() tmpReqPropsObj.setMimeType(fio.getFormat()) stagingFilename = ngamsHighLevelLib.\ genStagingFilename(srvObj.getCfg(), tmpReqPropsObj, trgDiskInfo, fio.getFileId()) # Receive the data into the Staging File using the urllib. if (srvObj.getHostId() != hostId): # Example: http://host:7777/RETRIEVE?file_id=id&file_version=1 ipAddress = hostInfoDic[hostId].getIpAddress() portNo = hostInfoDic[hostId].getSrvPort() fileUrl = "http://" + ipAddress + ":" + str(portNo) +\ "/RETRIEVE?" + "file_id=" + fio.getFileId() +\ "&file_version=" + str(fio.getFileVersion()) # If a specific Disk ID for the source file is given, append # this. if (fio.getDiskId()): fileUrl += "&disk_id=%s" % fio.getDiskId() # Check if host is suspended, if yes, wake it up. if (srvObj.getDb().getSrvSuspended(hostId)): logger.debug("Clone Request - Waking up suspended " +\ "NGAS Host: %s", hostId) ngamsSrvUtils.wakeUpHost(srvObj, hostId) else: fileUrl = "file:" + mtPt + "/" + fio.getFilename() logger.debug("Receiving file via URI: %s into staging filename: %s", fileUrl, stagingFilename) # We try up to 5 times to retrieve the file in case a problem is # encountered during cloning. for attempt in range(5): try: filename, headers = urlrequest.urlretrieve(fileUrl, stagingFilename) _checkFile(srvObj, fio, stagingFilename, headers, checkChecksum) # If we get to this point the transfer was (probably) OK. break except Exception as e: rmFile(stagingFilename) errMsg = "Problem occurred while cloning file "+\ "via URL: " + fileUrl + " - Error: " + str(e) if (attempt < 4): errMsg += " - Retrying in 5s ..." logger.error(errMsg) time.sleep(5) else: raise Exception(errMsg) # We simply copy the file into the same destination as the # source file (but on another disk). targPathName = os.path.dirname(fio.getFilename()) targFilename = os.path.basename(fio.getFilename()) complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\ "/" + targPathName) checkCreatePath(complTargPath) complFilename = os.path.normpath(complTargPath + "/"+targFilename) mvTime = mvFile(stagingFilename, complFilename) ngamsLib.makeFileReadOnly(complFilename) # Update status for new file in the DB. newFileInfo = fio.clone().setDiskId(trgDiskInfo.getDiskId()).\ setCreationDate(getFileCreationTime(complFilename)) fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fio.getFileId(), fio.getFileVersion()) newFileInfo.write(srvObj.getHostId(), srvObj.getDb()) # Update status for the Target Disk in DB + check if the disk is # completed. if (fileExists): mvTime = 0 dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\ setDiskId(trgDiskInfo.getDiskId()).\ setFileExists(fileExists).\ setFileSize(fio.getFileSize()).setIoTime(mvTime) ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj) ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId()) # Update the clone file status list. if (emailNotif): tmpFileList.setStatus(NGAMS_SUCCESS) tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE")) tmpFileList.addFileInfoObj(newFileInfo.setTag("TARGET_FILE")) cloneStatusDbm.addIncKey(tmpFileList) successCloneCount += 1 # If running as a cache archive, update the Cache New Files DBM # with the information about the new file. if (srvObj.getCachingActive()): diskId = trgDiskInfo.getDiskId() fileId = fio.getFileId() fileVer = fio.getFileVersion() filename = fio.getFilename() ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId, fileId, fileVer, filename) # Generate a confirmation log entry. cloneTime = time.time() - clone_start timeAccu += cloneTime msg = genLog("NGAMS_INFO_FILE_CLONED", [fio.getFileId(), fio.getFileVersion(), fio.getDiskId(), hostId]) msg = msg + ". Time: %.3fs. Total time: %.3fs." %\ (cloneTime, timeAccu) logger.info(msg, extra={'to_syslog': True}) except Exception as e: cloneTime = time.time() - clone_start timeAccu += cloneTime errMsg = genLog("NGAMS_ER_FILE_CLONE_FAILED", [fio.getFileId(), fio.getFileVersion(), fio.getDiskId(), hostId, str(e)]) if (abortCloneLoop): logger.error(errMsg, extra={'to_syslog': True}) return else: logger.warning(errMsg) if (emailNotif): tmpFileList.setStatus(NGAMS_FAILURE + ": Error: " + errMsg) tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE")) cloneStatusDbm.addIncKey(tmpFileList) failedCloneCount += 1 # Delete Staging File if already created. if ((stagingFilename != "") and (os.path.exists(stagingFilename))): rmFile(stagingFilename) # Calculate time statistics. if (reqPropsObj): ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj.\ incActualCount(1), timeAccu) # Final update of the Request Status. if (reqPropsObj): complPercent = (100.0 * (float(reqPropsObj.getActualCount()) / float(reqPropsObj.getExpectedCount()))) reqPropsObj.setCompletionPercent(complPercent, 1) reqPropsObj.setCompletionTime(1) srvObj.updateRequestDb(reqPropsObj) # Send Clone Report with list of files cloned to a possible # requestor(select) of this. totFiles = (successCloneCount + failedCloneCount) if (emailNotif): xmlStat = 0 # TODO: Generation of XML status report is disabled since we cannot # handle for the moment XML documents with 1000s of elements. if (xmlStat): cloneStatusFileList = ngamsFileList.\ ngamsFileList("FILE_CLONING_STATUS_REPORT", "File Cloning Status Report") fileCount = 0 while (fileCount < cloneStatusDbm.getCount()): tmpFileList = cloneStatusDbm.get(str(fileCount)) cloneStatusFileList.addFileListObj(tmpFileList) # Make overall status. cloneStatusFileList.setStatus("SUCCESS: " +\ str(successCloneCount) +\ ", FAILURE: " +\ str(failedCloneCount) +\ ", NOT DONE: " +\ str(len(cloneStatusFileList) -\ successCloneCount -\ failedCloneCount)) status = srvObj.genStatus(NGAMS_SUCCESS, "CLONE command status report").\ addFileList(cloneStatusFileList) statRep = status.genXmlDoc(0, 0, 0, 1, 0) statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep) mimeType = NGAMS_XML_MT else: # Generate a 'simple' ASCII report. statRep = tmpFilePat + "_NOTIF_EMAIL.txt" fo = open(statRep, "w") if (reqPropsObj.hasHttpPar("disk_id")): diskId = reqPropsObj.getHttpPar("disk_id") else: diskId = "-----" if (reqPropsObj.hasHttpPar("file_id")): fileId = reqPropsObj.getHttpPar("file_id") else: fileId = "-----" if (reqPropsObj.hasHttpPar("file_version")): fileVersion = reqPropsObj.getHttpPar("file_version") else: fileVersion = "-----" tmpFormat = "CLONE STATUS REPORT:\n\n" +\ "==Summary:\n\n" +\ "Date: %s\n" +\ "NGAS Host: %s\n" +\ "Disk ID: %s\n" +\ "File ID: %s\n" +\ "File Version: %s\n" +\ "Total Number of Files: %d\n" +\ "Number of Cloned Files: %d\n" +\ "Number of Failed Files: %d\n" +\ "Total processing time (s): %.3f\n" +\ "Handling time per file (s): %.3f\n\n" +\ "==File List:\n\n" fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), diskId, fileId, str(fileVersion), totFiles, successCloneCount, failedCloneCount, timeAccu, (timeAccu / totFiles))) tmpFormat = "%-70s %-70s %-7s\n" fo.write(tmpFormat % ("Source File", "Target File", "Status")) fo.write(tmpFormat % (70 * "-", 70 * "-", 7 * "-")) key = 1 while (1): if (not cloneStatusDbm.hasKey(str(key))): break tmpFileList = cloneStatusDbm.get(str(key)) key += 1 srcFileObj = tmpFileList.getFileInfoObjList()[0] srcFile = "%s/%s/%d" % (srcFileObj.getDiskId(), srcFileObj.getFileId(), srcFileObj.getFileVersion()) if (tmpFileList.getStatus() == NGAMS_SUCCESS): trgFileObj = tmpFileList.getFileInfoObjList()[1] trgFile = "%s/%s/%d" % (trgFileObj.getDiskId(), trgFileObj.getFileId(), trgFileObj.getFileVersion()) else: trgFile = "-----" fo.write(tmpFormat % (srcFile,trgFile,tmpFileList.getStatus())) fo.write(149 * "-") fo.write("\n\n==END\n") fo.close() mimeType = NGAMS_TEXT_MT # Send out the status report. emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",") attachmentName = "CloneStatusReport" if (reqPropsObj.hasHttpPar("disk_id")): attachmentName += "-" + reqPropsObj.getHttpPar("disk_id") if (reqPropsObj.hasHttpPar("file_id")): attachmentName += "-" + reqPropsObj.getHttpPar("file_id") if (reqPropsObj.hasHttpPar("file_version")): attachmentName += "-" + reqPropsObj.getHttpPar("file_version") ngamsNotification.notify(srvObj.host_id, srvObj.cfg, NGAMS_NOTIF_INFO, "CLONE STATUS REPORT", statRep, recList=emailAdrList, force=1, contentType=mimeType, attachmentName=attachmentName) del cloneStatusDbm rmFile(cloneStatusDbmName + "*") rmFile(statRep) if (cloneListDbm): del cloneListDbm rmFile(cloneListDbmName + "*") logger.info("_cloneExec(). Total time: %.3fs. Average time per file: %.3fs.", timeAccu, (timeAccu / totFiles))
def _cloneExec(srvObj, cloneListDbmName, tmpFilePat, targetDiskId, reqPropsObj): """ See documentation of ngamsCloneCmd._cloneThread(). This function is merely implemented in order to encapsulate the whole process to be able to clean up properly when the processing is terminated. """ T = TRACE(1) cloneListDbm = cloneStatusDbm = None emailNotif = 0 checkChecksum = 1 if (reqPropsObj): if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1 if (reqPropsObj.hasHttpPar("check")): checkChecksum = int(reqPropsObj.getHttpPar("check")) # Open clone list DB. cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName) # We have to get the port numbers of the hosts where the files to be # cloned are stored. hostInfoDic = {} cloneListDbm.initKeyPtr() while (1): key, fileInfo = cloneListDbm.getNext() if (not key): break hostInfoDic[fileInfo[1]] = -1 hostInfoDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), hostInfoDic.keys()) # The cloning loop. Loop over the list of files to clone and generate # a report with the result. if (emailNotif): cloneStatusDbmName = tmpFilePat + "_CLONE_STATUS_DB" cloneStatusDbm = ngamsDbm.ngamsDbm(cloneStatusDbmName, cleanUpOnDestr=0, writePerm=1) successCloneCount = 0 failedCloneCount = 0 abortCloneLoop = 0 timeAccu = 0.0 key = 0 while (1): clone_start = time.time() if (not cloneListDbm.hasKey(str(key))): break fileInfo = cloneListDbm.get(str(key)) key += 1 # Check if we have permission to run. Otherwise, stop. if (not srvObj.getThreadRunPermission()): break fio = fileInfo[0] mtPt = fileInfo[2] if (emailNotif): tmpFileList = ngamsFileList.\ ngamsFileList("FILE_CLONE_STATUS", "File: " + fio.getFileId() + "/" +\ fio.getDiskId() + "/" +\ str(fio.getFileVersion())) hostId = fileInfo[1] text = "Cloning file - File ID: %s/%d, on disk " +\ "with ID: %s on host: %s" logger.debug(text, fio.getFileId(), fio.getFileVersion(), fio.getDiskId(), hostId) # We generate a local Staging File and archive this. stagingFilename = "" try: # Check if file is marked as bad. if (fio.getFileStatus()[0] == "1"): errMsg = "File marked as bad - skipping!" raise Exception, errMsg if (targetDiskId == ""): # Try to find a disk not hosting already a file with that # ID + version. diskExemptList = [fio.getDiskId()] while (1): trgDiskInfo = ngamsDiskUtils.\ findTargetDisk(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), fio.getFormat(), 1, diskExemptList) # Check if a file with that ID + version is already # stored on the selected Target Disk. if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fio.getFileId(), fio.getFileVersion())): # This file is already stored on the given disk. # Add to the exempt list. diskExemptList.append(trgDiskInfo.getDiskId()) else: # OK, this disk should be OK, stop looking for a # suitable Target Disk. break else: try: trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\ read(srvObj.getDb(), targetDiskId) slotId = trgDiskInfo.getSlotId() storageSetId = srvObj.getCfg().\ getStorageSetFromSlotId(slotId).\ getStorageSetId() trgDiskInfo.setStorageSetId(storageSetId) except Exception: abortCloneLoop = 1 raise # We don't accept to clone onto the same disk (this would mean # overwriting). if (trgDiskInfo.getDiskId() == fio.getDiskId()): err = "Source and target files are identical" msg = "Failed in cloning file with ID: " + fio.getFileId() +\ "/Version: " + str(fio.getFileVersion()) +\ " on disk with ID: " + fio.getDiskId() +\ " on host: " + hostId + ". Reason: " + err logger.warning(msg) if (emailNotif): tmpFileList.setStatus(NGAMS_FAILURE + ": " + err) tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE")) cloneStatusDbm.addIncKey(tmpFileList) failedCloneCount += 1 continue tmpReqPropsObj = ngamsReqProps.ngamsReqProps() tmpReqPropsObj.setMimeType(fio.getFormat()) stagingFilename = ngamsHighLevelLib.\ genStagingFilename(srvObj.getCfg(), tmpReqPropsObj, trgDiskInfo, fio.getFileId()) # Receive the data into the Staging File using the urllib. if (srvObj.getHostId() != hostId): # Example: http://host:7777/RETRIEVE?file_id=id&file_version=1 ipAddress = hostInfoDic[hostId].getIpAddress() portNo = hostInfoDic[hostId].getSrvPort() fileUrl = "http://" + ipAddress + ":" + str(portNo) +\ "/RETRIEVE?" + "file_id=" + fio.getFileId() +\ "&file_version=" + str(fio.getFileVersion()) # If a specific Disk ID for the source file is given, append # this. if (fio.getDiskId()): fileUrl += "&disk_id=%s" % fio.getDiskId() # Check if host is suspended, if yes, wake it up. if (srvObj.getDb().getSrvSuspended(hostId)): logger.debug("Clone Request - Waking up suspended " +\ "NGAS Host: %s", hostId) ngamsSrvUtils.wakeUpHost(srvObj, hostId) else: fileUrl = "file:" + mtPt + "/" + fio.getFilename() logger.debug( "Receiving file via URI: %s into staging filename: %s", fileUrl, stagingFilename) # We try up to 5 times to retrieve the file in case a problem is # encountered during cloning. for attempt in range(5): try: filename, headers = urllib.urlretrieve( fileUrl, stagingFilename) _checkFile(srvObj, fio, stagingFilename, headers, checkChecksum) # If we get to this point the transfer was (probably) OK. break except Exception, e: rmFile(stagingFilename) errMsg = "Problem occurred while cloning file "+\ "via URL: " + fileUrl + " - Error: " + str(e) if (attempt < 4): errMsg += " - Retrying in 5s ..." logger.error(errMsg) time.sleep(5) else: raise Exception, errMsg # We simply copy the file into the same destination as the # source file (but on another disk). targPathName = os.path.dirname(fio.getFilename()) targFilename = os.path.basename(fio.getFilename()) complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\ "/" + targPathName) checkCreatePath(complTargPath) complFilename = os.path.normpath(complTargPath + "/" + targFilename) mvTime = mvFile(stagingFilename, complFilename) ngamsLib.makeFileReadOnly(complFilename) # Update status for new file in the DB. newFileInfo = fio.clone().setDiskId(trgDiskInfo.getDiskId()).\ setCreationDate(getFileCreationTime(complFilename)) fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fio.getFileId(), fio.getFileVersion()) newFileInfo.write(srvObj.getHostId(), srvObj.getDb()) # Update status for the Target Disk in DB + check if the disk is # completed. if (fileExists): mvTime = 0 dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\ setDiskId(trgDiskInfo.getDiskId()).\ setFileExists(fileExists).\ setFileSize(fio.getFileSize()).setIoTime(mvTime) ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj) ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId()) # Update the clone file status list. if (emailNotif): tmpFileList.setStatus(NGAMS_SUCCESS) tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE")) tmpFileList.addFileInfoObj(newFileInfo.setTag("TARGET_FILE")) cloneStatusDbm.addIncKey(tmpFileList) successCloneCount += 1 # If running as a cache archive, update the Cache New Files DBM # with the information about the new file. if (srvObj.getCachingActive()): diskId = trgDiskInfo.getDiskId() fileId = fio.getFileId() fileVer = fio.getFileVersion() filename = fio.getFilename() ngamsCacheControlThread.addEntryNewFilesDbm( srvObj, diskId, fileId, fileVer, filename) # Generate a confirmation log entry. cloneTime = time.time() - clone_start timeAccu += cloneTime msg = genLog("NGAMS_INFO_FILE_CLONED", [ fio.getFileId(), fio.getFileVersion(), fio.getDiskId(), hostId ]) msg = msg + ". Time: %.3fs. Total time: %.3fs." %\ (cloneTime, timeAccu) logger.info(msg, extra={'to_syslog': True}) except Exception, e: cloneTime = time.time() - clone_start timeAccu += cloneTime errMsg = genLog("NGAMS_ER_FILE_CLONE_FAILED", [ fio.getFileId(), fio.getFileVersion(), fio.getDiskId(), hostId, str(e) ]) if (abortCloneLoop): logger.error(errMsg, extra={'to_syslog': True}) thread.exit() else: logger.warning(errMsg) if (emailNotif): tmpFileList.setStatus(NGAMS_FAILURE + ": Error: " + errMsg) tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE")) cloneStatusDbm.addIncKey(tmpFileList) failedCloneCount += 1 # Delete Staging File if already created. if ((stagingFilename != "") and (os.path.exists(stagingFilename))): rmFile(stagingFilename)
def _locateArchiveFile(srvObj, fileId, fileVersion, diskId, hostId, reqPropsObj, files, include_compression): """ See description of ngamsFileUtils.locateArchiveFile(). This function is used simply to encapsulate the complete processing to be able to clean up. """ msg = "_locateArchiveFile() - Disk ID: %s - File ID: " +\ "%s - File Version: %d ..." logger.debug(msg, str(diskId), fileId, int(fileVersion)) # Filter out files not on specified host if host ID is given. if (hostId): files = filter(lambda x: x[1] == hostId, files) # If no file was found we raise an exception. if not files: tmpFileRef = fileId if (fileVersion > 0): tmpFileRef += "/Version: " + str(fileVersion) if (diskId): tmpFileRef += "/Disk ID: " + diskId if (hostId): tmpFileRef += "/Host ID: " + hostId errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [tmpFileRef]) raise Exception(errMsg) # We now sort the file information sub-lists in the file list. # The priori is as follows: # # 1. Local host. # 2. Same cluster. # 3. Same domain (e.g. hq.eso.org). # 4. Other files (remote files). localHostFileList = [] clusterFileList = [] domainFileList = [] remoteFileList = [] all_hosts = set([x[1] for x in files]) hostDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), all_hosts) # Loop over the candidate files and sort them. fileCount = idx = 0 for fileInfo in files: fileHost = fileInfo[1] if (hostDic[fileHost].getHostType() == NGAMS_HOST_LOCAL): localHostFileList.append(fileInfo) elif (hostDic[fileHost].getHostType() == NGAMS_HOST_CLUSTER): clusterFileList.append(fileInfo) elif (hostDic[fileHost].getHostType() == NGAMS_HOST_DOMAIN): domainFileList.append(fileInfo) else: # NGAMS_HOST_REMOTE: remoteFileList.append(fileInfo) idx += 1 fileCount += 1 # The highest priority of the file is determined by the File Version, # the latest version is preferred even though this may be stored # on another NGAS host. # A dictionary is built up, which contains the candidate files. The # format is such there each version found is one key. For each key # (= version) there is a list with the corresponding file information # order according to the location. candFileDic = {} fileLists = [[NGAMS_HOST_LOCAL, localHostFileList], [NGAMS_HOST_CLUSTER, clusterFileList], [NGAMS_HOST_DOMAIN, domainFileList], [NGAMS_HOST_REMOTE, remoteFileList]] for fileListInfo in fileLists: location = fileListInfo[0] fileList = fileListInfo[1] for fileInfo in fileList: fileVer = fileInfo[0].getFileVersion() # Create a list in connection with each File Version key. if fileVer not in candFileDic: candFileDic[fileVer] = [] candFileDic[fileVer].append([location, fileInfo[0], fileInfo[1]]) fileVerList = list(candFileDic) fileVerList.sort(reverse=True) if logger.level <= logging.DEBUG: msg = "" count = 1 for fileVer in fileVerList: for fi in candFileDic[fileVer]: msg += "(" + str(count) + ": Location:" + fi[0] +\ ", Host:" + fi[2] + ", Version:" +\ str(fi[1].getFileVersion()) + ") " count += 1 logger.debug("File list to check: " + msg) # If no files were found we raise an exception. if (len(candFileDic) == 0): if (fileVersion != -1): fileRef = fileId + "/V" + str(fileVersion) else: fileRef = fileId errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [fileRef]) raise Exception(errMsg) # We generate a list with the Disk IDs (which we need later). # Generate a dictionary with Disk Info Objects. diskIdDic = {} for fileVer in fileVerList: for fileInfo in candFileDic[fileVer]: diskIdDic[fileInfo[1].getDiskId()] = fileInfo[1].getDiskId() sqlDiskInfo = srvObj.getDb().getDiskInfoFromDiskIdList(list(diskIdDic)) diskInfoDic = {} for diskInfo in sqlDiskInfo: diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().unpackSqlResult(diskInfo) diskInfoDic[diskInfoObj.getDiskId()] = diskInfoObj logger.debug("Disk Info Objects Dictionary: %s", str(diskInfoDic)) # Check if the files are accessible - when the first accessible file # in the fileList is found, the information is returned as the file wanted. # To check the file accessibility, it is also checked if the NG/AMS # 'responsible' for the file, allows for Retrieve Requests (only done # in connection with a Retrieve Request). logger.debug("Checking which of the candidate files should be selected ...") foundFile = 0 for fileVer in fileVerList: if (foundFile): break for fileInfo in candFileDic[fileVer]: location = fileInfo[0] fileInfoObj = fileInfo[1] host = fileInfo[2] diskInfoObj = diskInfoDic[fileInfoObj.getDiskId()] port = hostDic[host].getSrvPort() logger.debug("Checking candidate file with ID: %s on host/port: %s/%s. " + \ "Location: %s", fileInfoObj.getFileId(), host, str(port), location) # If the file is stored locally we check if it is accessible, # otherwise we send a STATUS/file_access request to the # host in question. if (location == NGAMS_HOST_LOCAL): # Check first if the local system supports retrieve requests. # (if relevant). if (reqPropsObj): if (reqPropsObj.getCmd() == NGAMS_RETRIEVE_CMD): if (not srvObj.getCfg().getAllowRetrieveReq()): continue # Check if the file is accessible. filename = os.path.normpath(diskInfoObj.getMountPoint()+"/" +\ fileInfoObj.getFilename()) logger.debug("Checking if local file with name: %s is available", filename) if (not os.path.exists(filename)): logger.debug(genLog("NGAMS_INFO_FILE_NOT_AVAIL", [fileId, host])) else: logger.debug(genLog("NGAMS_INFO_FILE_AVAIL", [fileId, host])) foundFile = 1 break else: logger.debug("Checking if file with ID/Version: %s/%s " +\ "is available on host/port: %s/%s", fileInfoObj.getFileId(), str(fileInfoObj.getFileVersion()), host, str(port)) # If a server hosting a file is suspended, it is woken up # to be able to check if the file is really accessible. if (hostDic[host].getSrvSuspended() == 1): logger.debug("Server hosting requested file (%s/%s) is suspended " + \ "- waking up server ...", host, str(port)) try: ngamsSrvUtils.wakeUpHost(srvObj, host) logger.debug("Suspended server hosting requested file (%s/%s) " +\ "has been woken up", host, str(port)) except Exception: logger.exception("Error waking up server hosting selected " +\ "file") continue # The file is hosted on a host, which is not suspended or # which was successfully woken up. pars = [["file_access", fileInfoObj.getFileId()]] if (fileInfoObj.getFileVersion() != -1): pars.append(["file_version", fileInfoObj.getFileVersion()]) ipAddress = hostDic[host].getIpAddress() authHdr = ngamsSrvUtils.genIntAuthHdr(srvObj) resp = ngamsHttpUtils.httpGet(ipAddress, port, NGAMS_STATUS_CMD, pars=pars, auth=authHdr) with contextlib.closing(resp): data = resp.read() statusObj = ngamsStatus.ngamsStatus().unpackXmlDoc(data, 1) if logger.isEnabledFor(logging.DEBUG): logger.debug("Result of File Access Query: %s", re.sub("\n", "", str(statusObj.genXml().toprettyxml(' ', '\n')))) if ((statusObj.getMessage().\ find("NGAMS_INFO_FILE_AVAIL") == -1)): logger.debug(genLog("NGAMS_INFO_FILE_NOT_AVAIL", [fileId, host])) else: logger.debug(genLog("NGAMS_INFO_FILE_AVAIL", [fileId, host])) foundFile = 1 break # If no file was found we raise an exception. if (not foundFile): errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [fileId]) raise Exception(errMsg) # The file was found, get the info necessary for the acquiring the file. ipAddress = hostDic[host].getIpAddress() srcFileInfo = [location, host, ipAddress, port, diskInfoObj.getMountPoint(), fileInfoObj.getFilename(), fileInfoObj.getFileId(), fileInfoObj.getFileVersion(), fileInfoObj.getFormat()] if include_compression: srcFileInfo.append(fileInfoObj.getCompression()) msg = "Located suitable file for request - File ID: %s. " +\ "Info for file found - Location: %s - Host ID/IP: %s/%s - " +\ "Port Number: %s - File Version: %d - Filename: %s - " +\ "Mime-type: %s" logger.debug(msg, fileId, location, host, ipAddress, port, fileInfoObj.getFileVersion(), fileInfoObj.getFilename(), fileInfoObj.getFormat()) return srcFileInfo