def clone_check_file(disk_id, file_id, file_version, connection, ngas_client): """ Clone a file a check afterwards if it was successfully cloned :param disk_id: ID of disk cloned (string) :param file_id: ID of file cloned (string) :param file_version: Version of file to check (integer) :param connection: DB connection object (ngamsDb) :param ngas_client: Initiated instance of NG/AMS P-Client Object (ngamsPClient) :return: Updated Check Report (string) """ message = "\n-> Attempting to clone file: {:s}/{:s}/{:s}".format( disk_id, file_id, file_version) print(message) message += " - status: " result = ngas_client.clone(file_id, disk_id, file_version, wait=1) if result.getStatus() == "FAILURE": status = "FAILURE: " + str(result.get_message()) + "\n" else: # Check if file was really cloned result = connection.getFileInfoFromFileIdHostId( _get_target_host(), file_id, file_version) if not result: status = "FAILURE: File not cloned!\n" else: file_info = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(result) file_status_parameters = [["disk_id", file_info.get_disk_id()], ["file_access", file_id], ["file_version", file_version]] client = ngamsPClient.ngamsPClient(getHostName(), ngas_client.getPort()) result = client.status(file_status_parameters) status = result.get_message() + "\n" return message + status
def locateArchiveFile(srvObj, fileId, fileVersion=-1, diskId="", hostId="", reqPropsObj=None, include_compression=False): """ Locate the file indicated by the File ID. Returns a list containing the necessary information for retrieving the file: [<Location>, <File Host>, <IP Address>, <Port No>, <Mount Point>, <Filename>, <File ID>, <File Version>, <Mime-Type>] - whereby: <Location> = Location of the file (NGAMS_HOST_LOCAL, NGAMS_HOST_CLUSTER, NGAMS_HOST_DOMAIN, NGAMS_HOST_REMOTE). <File Host> = Host ID of host to be contacted to get access to the file. <IP Address> = IP Address of host to be contacted to get access to the file. <Port No> = Port number used by the NG/AMS Server. <Mount Point> = Mount point at which the file is residing. <Filename> = Name of file relative to mount point. <File ID> = ID of file. <File Version> = Version of file. <Mime-Type> = Mime-type of file (as registered in NGAS). srvObj: Reference to NG/AMS server class object (ngamsServer). fileId: File ID of file to locate (string). fileVersion: Version of the file (integer). diskId: ID of the disk hosting the file (string). hostId: ID of the host where the file is located (string). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps|None). Returns: List with information about file location (list). """ # Get a list with the candidate files matching the query conditions. res = srvObj.getDb().getFileInfoFromFileId(fileId, fileVersion, diskId, ignore=0, dbCursor=False) # r[-2] is the host_id, r[-1] is the mount point all_info = [] for r in res: file_info = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(r) all_info.append((file_info, r[-2], r[-1])) return _locateArchiveFile(srvObj, fileId, fileVersion, diskId, hostId, reqPropsObj, all_info, include_compression)
def _encFileInfo2Obj(dbConObj, dbSnapshot, encFileInfoDic): """ Convert an encoded file info from the snapshot into an NG/AMS File Info Object. dbConObj: DB connection object (ngamsDb). dbSnapshot: Open DB object (bsddb). encFileInfoDic: Dictionary containing the encoded file information (dictionary). Returns: NG/AMS File Info Object (ngamsFileInfo). """ sqlFileInfo = [] #for n in range (ngamsDbCore.NGAS_FILES_CREATION_DATE + 1): for n in range(ngamsDbCore.NGAS_FILES_CONTAINER_ID + 1): sqlFileInfo.append(None) idxKeys = encFileInfoDic.keys() for idx in idxKeys: kid = NGAMS_SN_SH_ID2NM_TAG + six.b(str(idx)) if (not dbSnapshot.has_key(kid)): logger.warning("dbSnapshot has no key '%s', is it corrupted?", str(kid)) return None colName = _readDb(dbSnapshot, kid) sqlFileInfoIdx = dbConObj.getNgasFilesMap()[colName] sqlFileInfo[sqlFileInfoIdx] = encFileInfoDic[idx] tmpFileInfoObj = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(sqlFileInfo) return tmpFileInfoObj
def test_FileVerHandling_01(self): """ Synopsis: Test that REMFILE removes proper File Version. Description: The purpose of the test is to verify that if several versions of a file are found in the archive, the proper version is deleted. Expected Result: The chosen version of the file, should be selected and should be removed. Test Steps: - Start server. - Archive a file 3 times into the archive. - Clone the file onto another disk. - Remove version 2 of the cloned file. Remarks: ... Test Data: ... """ _, dbObj = self.prepExtSrv(cfgProps=(('NgamsCfg.Db[1].Snapshot', "0"), )) for _ in range(3): self.archive("src/SmallFile.fits") diskId1 = self.ngas_disk_id("FitsStorage1/Main/1") fileId = "TEST.2001-05-08T15:25:00.123" fileVer = 2 self.get_status(NGAMS_CLONE_CMD, pars=[["disk_id", diskId1]]) diskId2 = self.ngas_disk_id("FitsStorage2/Main/3") filePath = self.ngas_path("FitsStorage2-Main-3/saf/" +\ "2001-05-08/2/TEST.2001-05-08T15:25:00.123.fits.gz") for execute in [0, 1]: httpPars = [["disk_id", diskId2], ["file_id", fileId], ["file_version", fileVer], ["execute", execute]] self.get_status(NGAMS_REMFILE_CMD, pars=httpPars) fileInfo = ngamsFileInfo.ngamsFileInfo() try: fileInfo.read(getHostName(), dbObj, fileId, fileVer, diskId2) except: pass if (execute == 0): if (not os.path.exists(filePath)): self.fail("File removed unexpectedly") else: if (os.path.exists(filePath)): self.fail("File not removed as expected") refStatFile = "ref/ngamsRemFileCmdTest_test_" +\ "FileVerHandling_01_0%d_ref" % (execute + 3) msg = "Incorrect status for REMFILE Command/execution" self.assert_status_ref_file(refStatFile, fileInfo, msg=msg)
def test_RegisterCmd_1(self): """ Synopsis: REGISTER Command/register single file compl. path. Description: Test handling of the REGISTER Command under normal circumstances. Expected Result: The REGISTER Command should be accepted by the server and should be executed successfully. Test Steps: - Start server. - Copy file onto NGAS Disk. - Submit REGISTER Command requesting to register the file copied over (wait=1). - Check response from the server that the request was successfully executed. - Check the DB info for the registered file. Remarks: ... """ _, dbObj = self.prepExtSrv() srcFile = "src/SmallFile.fits" tmpSrcFile = "/tmp/ngamsTest/NGAS/" +\ "FitsStorage2-Main-3/saf/test/SmallFile.fits" checkCreatePath(os.path.dirname(tmpSrcFile)) shutil.copy(srcFile, tmpSrcFile) tmpStatFile = sendExtCmd(8888, NGAMS_REGISTER_CMD, [["path", tmpSrcFile]]) refStatFile = "ref/ngamsRegisterCmdTest_test_RegisterCmd_1_ref" self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect status returned for REGISTER command") diskId = "tmp-ngamsTest-NGAS-FitsStorage2-Main-3" filePrefix = "ngamsRegisterCmdTest_test_RegisterCmd_1" fileInfoRef = "ref/" + filePrefix + "_FileInfo_ref" fileInfoTmp = "tmp/" + filePrefix + "_FileInfo_tmp" fileId = "TEST.2001-05-08T15:25:00.123" startTime = time.time() host_id = getHostName() + ":8888" while ((time.time() - startTime) < 10): tmpFileRes = dbObj.getFileInfoFromFileIdHostId( host_id, fileId, 1, diskId) if (tmpFileRes): break if not tmpFileRes: self.fail( "Couldn't get fileInfo result from database within 10 seconds") tmpFileObj = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(tmpFileRes) saveInFile(fileInfoTmp, filterDbStatus1(tmpFileObj.dumpBuf())) self.checkFilesEq(fileInfoRef, fileInfoTmp, "Incorrect info in DB for registered file")
def test_get_file_info_list_with_wildcards(self): """Double-check that wildcards in fileId work""" disk_info = ngamsDiskInfo.ngamsDiskInfo() disk_info.setDiskId('disk-id') disk_info.write(self.db) file_info = ngamsFileInfo.ngamsFileInfo() file_info.setDiskId('disk-id') file_info.setFileId('file-id') file_info.write('host-id', self.db, genSnapshot=0) res = list(self.db.getFileInfoList('disk-id', fileId="*")) self.assertEqual(1, len(res))
def receiveData(srvObj, reqPropsObj, httpRef): """ Receive the data in connection with the Rearchive Request. For a description of the parameters: Check handleCmd(). Returns: Tuple with File Info Object for the file to be rearchived and Disk Info Object for the selected target disk (tuple/(fileInfo, ngamsDiskInfo)). """ # Note, this algorithm does not implement support for back-log buffering # for speed optimization reasons. # Unpack the file information contained in the HTTP header # (NGAS-File-Info). encFileInfo = reqPropsObj.getHttpHdr(NGAMS_HTTP_HDR_FILE_INFO) if (not encFileInfo): msg = "Error. Must provide NGAS File Information " +\ "(RFC 3548 encoded) in HTTP header NGAS-File-Info" raise Exception, msg fileInfoXml = base64.b64decode(encFileInfo) fileInfoObj = ngamsFileInfo.ngamsFileInfo().unpackXmlDoc(fileInfoXml) # Find the most suitable target Disk Set. trgDiskInfoObj = None diskExList = [] while (True): try: tmpTrgDiskInfo = ngamsDiskUtils.\ findTargetDisk(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), reqPropsObj.getMimeType(), sendNotification = 0, diskExemptList = diskExList, reqSpace = reqPropsObj.getSize()) if (srvObj.getDb().fileInDb(tmpTrgDiskInfo.getDiskId(), fileInfoObj.getFileId(), fileInfoObj.getFileVersion())): diskExList.append(tmpTrgDiskInfo.getDiskId()) else: trgDiskInfoObj = tmpTrgDiskInfo break except Exception, e: msg = "Error locating target disk for REARCHIVE Command. Error: %s" msg = msg % str(e) raise Exception, msg
def test_RegisterCmd_1(self): """ Synopsis: REGISTER Command/register single file compl. path. Description: Test handling of the REGISTER Command under normal circumstances. Expected Result: The REGISTER Command should be accepted by the server and should be executed successfully. Test Steps: - Start server. - Copy file onto NGAS Disk. - Submit REGISTER Command requesting to register the file copied over (wait=1). - Check response from the server that the request was successfully executed. - Check the DB info for the registered file. Remarks: ... """ _, db_obj = self.prepExtSrv() status = self.copy_and_register() msg = "Incorrect status returned for REGISTER command" ref_stat_file = "ref/ngamsRegisterCmdTest_test_RegisterCmd_1_ref" self.assert_status_ref_file(ref_stat_file, status, msg=msg) disk_id = self.ngas_disk_id("FitsStorage2/Main/3") file_prefix = "ngamsRegisterCmdTest_test_RegisterCmd_1" file_info_ref = "ref/" + file_prefix + "_FileInfo_ref" file_id = "TEST.2001-05-08T15:25:00.123" start_time = time.time() host_id = getHostName() + ":8888" while (time.time() - start_time) < 10: tmp_file_res = db_obj.getFileInfoFromFileIdHostId( host_id, file_id, 1, disk_id) if tmp_file_res: break if not tmp_file_res: self.fail( "Couldn't get fileInfo result from database within 10 seconds") tmp_file_obj = ngamsFileInfo.ngamsFileInfo().unpackSqlResult( tmp_file_res) msg = "Incorrect info in DB for registered file" self.assert_status_ref_file(file_info_ref, tmp_file_obj, msg=msg)
def rearchive(self, fileUri, fileInfoXml, pars=[]): """ Send a Re-archive Request to the associated NG/AMS Server asking to have the file specified by the URI archived. This can either be a Re-archive Push Request or a Re-archive Pull Request. fileUri: URI of file to archive (string). fileInfoXml: NG/AMS XML File Information for the source file used as reference (string/XML). pars: Extra parameters to submit with the request. Must be in the format: [[<Par>, <Val>], [<Par>, <Val>], ...] (list). Returns: NG/AMS Status object (ngamsStatus). """ baseName = os.path.basename(fileUri) logger.info("Re-archiving file with URI: %s", baseName) pars = list(pars) if is_known_pull_url(fileUri): tmpFileInfo = ngamsFileInfo.ngamsFileInfo().\ unpackXmlDoc(fileInfoXml) encFileInfo = base64.b64encode(fileInfoXml) pars.append([NGAMS_HTTP_PAR_FILENAME, fileUri]) httpHdrs = { NGAMS_HTTP_HDR_FILE_INFO: encFileInfo, NGAMS_HTTP_HDR_CONTENT_TYPE: tmpFileInfo.getFormat() } res = self.get_status(NGAMS_REARCHIVE_CMD, pars=pars, hdrs=httpHdrs) else: msg = "Rearchive Push is not yet supported!" raise Exception(msg) return res
def _registerExec(srvObj, fileListDbmName, tmpFilePat, diskInfoDic, reqPropsObj=None): """ Register the files listed in the File List DBM (ngamsDbm), which match the mime-type(s) either specified in the 'mimeType' parameter, or if this is not specified, which match all the mime-types specified in the configuration file. When the registration procedure has been executed, the function sends an Email Notification message indicating which files were registered if the HTTP parameter 'notif_email' is given. The functions creates a File Info Objects per file handled and writes this in a temporary file, which is a DBM file. The keys in this DB is simply the file number in the sequence of files handled, pointing to a pickled ngamsFileInfo object. Each of the File Info Objects indicates if the file was registered or not. This is done by setting the tag of the File Info Object to one of the following values: REGISTERED: The file was successfully registered in the NGAS DB FAILED: The file was selected for registration but could not be properly registered because of inconsistencies. The status will be of the format: 'FAILED[: <reason>]'. REJECTED: A file found under the specified path directory was not accepted for cloning, usually because the mime-type was not correct. The status will be of the format: 'REJECTED[: <reason>]'. Note, that registration is comparable to archiving of files. For that reason a DAPI must be provided for each type of file that should be registered. If this is not fullfilled, the file registration will fail. Only files stored on one of the NGAS disks configured in the configuration file, are considered. Files stored in other locations are ignored. srvObj: Instance of NG/AMS Server object (ngamsServer). fileListDbmName: Name of a DBM containing the information about the files to be registered. Each element in the list is referred to by a key, which is a number. These points to a pickled list for each file containing the following information: [<Filename>, <Disk ID>, <Mime-Type>] The information for each disk concerned is also contained in the DB referred to by its Disk ID and by its mount point. The data is a pickled instance of the ngamsDiskInfo class (string). tmpFilePat: Pattern for temporary files used during the registration process (string). diskInfoDic: Dictionary with Disk IDs as keys pointing to the info about the disk (dictionary/ngamsDiskInfo). reqPropsObj: If an NG/AMS Request Properties Object is given, the Request Status will be updated as the request is carried out (ngamsReqProps). Returns: Void. """ emailNotif = 0 if (reqPropsObj): if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1 # Create the temporary BSD DB to contain the information for the # Email Notification Message. if (emailNotif): regDbmName = tmpFilePat + "_NOTIF_EMAIL" regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm=1) # Open the DBM containing the list of files to (possibly) register. fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1) # Want to parse files in alphabetical order. # TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting. tmpFileList = tmpFilePat + "_FILE_LIST" rmFile(tmpFileList) with open(tmpFileList, "wb") as fo: fileListDbm.initKeyPtr() while (1): dbmKey, fileInfo = fileListDbm.getNext() if (not dbmKey): break fo.write(dbmKey + b"\n") sortFileList = tmpFilePat + "_SORT_FILE_LIST" rmFile(sortFileList) shellCmd = "sort %s > %s" % (tmpFileList, sortFileList) stat, out, err = ngamsCore.execCmd(shellCmd) if (stat != 0): raise Exception("Error executing command: %s. Error: %s, %s" %\ (shellCmd, str(out), str(err))) rmFile(tmpFileList) # Go through each file in the list, check if the mime-type is among the # ones, which apply for registration. If yes try to register the file # by invoking the corresponding DAPI on the file. fileRegCount = 0 fileFailCount = 0 fileRejectCount = 0 regTimeAccu = 0.0 fileCount = 0 fo = open(sortFileList) run = 1 while (run): reg_start = time.time() dbmKey = fo.readline() if (dbmKey.strip() == ""): run = 0 continue fileInfo = fileListDbm.get(dbmKey[0:-1]) filename = fileInfo[0] diskId = fileInfo[1] mimeType = fileInfo[2] # Register the file. Check first, that exactly this file is # not already registered. In case it is, the file will be rejected. regPi = srvObj.getCfg().register_plugins[mimeType] logger.debug("Plugin found for %s: %s", mimeType, regPi) params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars) tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\ setMimeType(mimeType).\ setStagingFilename(filename).\ setTargDiskInfo(diskInfoDic[diskId]).\ setHttpMethod(NGAMS_HTTP_GET).\ setCmd(NGAMS_REGISTER_CMD).\ setSize(os.path.getsize(filename)).\ setFileUri(filename).\ setNoReplication(1) tmpFileObj = ngamsFileInfo.ngamsFileInfo() try: # Invoke Registration Plug-In. piName = regPi.name plugInMethod = loadPlugInEntryPoint(piName) piRes = plugInMethod(srvObj, tmpReqPropsObj, params) del tmpReqPropsObj # Check if this file is already registered on this disk. In case # yes, it is not registered again. files = srvObj.db.getFileSummary1(srvObj.getHostId(), [piRes.getDiskId()], [piRes.getFileId()]) fileRegistered = 0 for tmpFileInfo in files: tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT] tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME] tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\ tmpFilename) if (tmpComplFilename == filename): fileRegistered = 1 break if (fileRegistered): fileRejectCount += 1 tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\ "and path: %s is already registered on disk " +\ "with Disk ID: %s" tmpMsg = tmpMsgForm % (piRes.getFileId(), piRes.getFileVersion(), filename, piRes.getDiskId()) logger.warning(tmpMsg + ". File is not registered again.") if (emailNotif): tmpFileObj.\ setDiskId(diskId).setFilename(filename).\ setTag(tmpMsg) regDbm.addIncKey(tmpFileObj) if (reqPropsObj): reqPropsObj.incActualCount(1) ngamsHighLevelLib.stdReqTimeStatUpdate( srvObj, reqPropsObj, regTimeAccu) regTimeAccu += time.time() - reg_start fileCount += 1 continue # Calculate checksum. We maintain the old name for backwards # compatibility crc_variant = srvObj.cfg.getCRCVariant() if crc_variant == ngamsFileUtils.CHECKSUM_CRC32_INCONSISTENT: crc_variant = 'ngamsGenCrc32' checksum = ngamsFileUtils.get_checksum(65536, filename, crc_variant) or '' # Move file and update information about file in the NGAS DB. mvFile(filename, piRes.getCompleteFilename()) ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum, crc_variant) ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes) ngamsLib.makeFileReadOnly(piRes.getCompleteFilename()) if (emailNotif): uncomprSize = piRes.getUncomprSize() ingestDate = time.time() creDateSecs = getFileCreationTime(filename) tmpFileObj.\ setDiskId(diskId).\ setFilename(filename).\ setFileId(piRes.getFileId()).\ setFileVersion(piRes.getFileVersion()).\ setFormat(piRes.getFormat()).\ setFileSize(piRes.getFileSize()).\ setUncompressedFileSize(uncomprSize).\ setCompression(piRes.getCompression()).\ setIngestionDate(ingestDate).\ setIgnore(0).\ setChecksum(checksum).\ setChecksumPlugIn(crc_variant).\ setFileStatus(NGAMS_FILE_STATUS_OK).\ setCreationDate(creDateSecs).\ setTag("REGISTERED") fileRegCount += 1 # If running as a cache archive, update the Cache New Files DBM # with the information about the new file. if (srvObj.getCachingActive()): fileVer = fio.getFileVersion() ngamsCacheControlThread.addEntryNewFilesDbm( srvObj, diskId, piRes.getFileId(), fileVer, filename) # Generate a confirmation log entry. msg = genLog("NGAMS_INFO_FILE_REGISTERED", [ filename, piRes.getFileId(), piRes.getFileVersion(), piRes.getFormat() ]) time.sleep(0.005) regTime = time.time() - reg_start msg = msg + ". Time: %.3fs." % (regTime) logger.info(msg, extra={'to_syslog': 1}) except Exception as e: errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)]) logger.error(errMsg) if (emailNotif): tmpFileObj.\ setDiskId(diskId).setFilename(filename).\ setTag(errMsg) fileFailCount += 1 regTime = time.time() - reg_start # TODO (rtobar, 2016-01): Why don't we raise an exception here? # Otherwise the command appears as successful on the # client-side # Add the file information in the registration report. if (emailNotif): regDbm.addIncKey(tmpFileObj) # Update request status time information. regTimeAccu += regTime if (reqPropsObj): reqPropsObj.incActualCount(1) ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj, regTimeAccu) fileCount += 1 fo.close() rmFile(sortFileList) if (emailNotif): regDbm.sync() del fileListDbm rmFile(fileListDbmName + "*") # Final update of the Request Status. if (reqPropsObj): if (reqPropsObj.getExpectedCount() and reqPropsObj.getActualCount()): complPercent = (100.0 * (float(reqPropsObj.getActualCount()) / float(reqPropsObj.getExpectedCount()))) else: complPercent = 100.0 reqPropsObj.setCompletionPercent(complPercent, 1) reqPropsObj.setCompletionTime(1) srvObj.updateRequestDb(reqPropsObj) # Send Register Report with list of files cloned to a possible # requestor(select) of this. if (emailNotif): xmlStat = 0 if (xmlStat): # Create an instance of the File List Class, used to store the # Registration Report. regStat = ngamsFileList.\ ngamsFileList("FILE_REGISTRATION_STATUS", "Status report for file " +\ "registration") # Loop over the file objects in the BSD DB and add these # in the status object. regDbm.initKeyPtr() while (1): key, tmpFileObj = regDbm.getNext() if (not key): break regStat.addFileInfoObj(tmpFileObj) # Set overall status of registration procedure. regStat.setStatus("Files Found: " + str(fileCount + 1) + ", "+\ "Files Registered: " + str(fileRegCount) +\ ", " +\ "Files Failed: " + str(fileFailCount) + ", " +\ "Files Rejected: " + str(fileRejectCount)) status = srvObj.genStatus(NGAMS_SUCCESS, "REGISTER command status report").\ addFileList(regStat) statRep = status.genXmlDoc() statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep) mimeType = NGAMS_XML_MT else: # Generate a 'simple' ASCII report. statRep = tmpFilePat + "_NOTIF_EMAIL.txt" fo = open(statRep, "w") if (reqPropsObj): path = reqPropsObj.getHttpPar("path") else: path = "-----" if (fileCount): timePerFile = (regTimeAccu / fileCount) else: timePerFile = 0 tmpFormat = "REGISTER STATUS REPORT:\n\n" +\ "==Summary:\n\n" +\ "Date: %s\n" +\ "NGAS Host: %s\n" +\ "Search Path: %s\n" +\ "Total Number of Files: %d\n" +\ "Number of Registered Files: %d\n" +\ "Number of Failed Files: %d\n" +\ "Number of Rejected Files: %d\n" +\ "Total processing time (s): %.3f\n" +\ "Handling time per file (s): %.3f\n\n" +\ "==File List:\n\n" fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), path, fileCount, fileRegCount, fileFailCount, fileRejectCount, regTimeAccu, timePerFile)) tmpFormat = "%-80s %-32s %-3s %-10s\n" fo.write(tmpFormat %\ ("Filename", "File ID", "Ver", "Status")) fo.write(tmpFormat % (80 * "-", 32 * "-", 3 * "-", 10 * "-")) regDbm.initKeyPtr() while (1): key, tmpFileObj = regDbm.getNext() if (not key): break mtPt = diskInfoDic[tmpFileObj.getDiskId()].getMountPoint() filename = os.path.normpath(mtPt + "/" +\ tmpFileObj.getFilename()) line = tmpFormat %\ (filename, tmpFileObj.getFileId(), str(tmpFileObj.getFileVersion()), tmpFileObj.getTag()) fo.write(line) fo.write(128 * "-") fo.write("\n\n==END\n") fo.close() mimeType = NGAMS_TEXT_MT # Send out the status report. emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",") attachmentName = "RegisterStatusReport" if (reqPropsObj.hasHttpPar("path")): attachmentName += "-" + reqPropsObj.getHttpPar("path").\ replace("/", "_") ngamsNotification.notify(srvObj.host_id, srvObj.cfg, NGAMS_NOTIF_INFO, "REGISTER STATUS REPORT", statRep, recList=emailAdrList, force=1, contentType=mimeType, attachmentName=attachmentName) del regDbm rmFile(regDbmName + "*") rmFile(statRep) # Generate final status log + exit. if (fileCount > 0): timePerFile = (regTimeAccu / fileCount) else: timePerFile = 0.0 msg = "Registration procedure finished processing Register Request - " + \ "terminating. Files handled: %d. Total time: %.3fs. " + \ "Average time per file: %.3fs." logger.debug(msg, fileCount, regTimeAccu, timePerFile)
def _clone(srvObj, diskId, fileId, fileVersion, targetDiskId, reqPropsObj, httpRef, tmpFilePat): """ Internal function used by ngamsCloneCmd.clone() to carry out the cloning. See documentation for ngamsCloneCmd.clone(). """ targetDiskId = targetDiskId.strip() logger.debug("Handling file cloning with parameters - File ID: %s -" + \ "Disk ID: %s - File Version: %s - Target Disk ID: |%s|", fileId, diskId, str(fileVersion), targetDiskId) if (((fileId == "") and (diskId == "") and (fileVersion != -1)) or ((fileId == "") and (diskId == "") and (fileVersion == -1))): errMsg = genLog("NGAMS_ER_CMD_SYNTAX", [NGAMS_CLONE_CMD, "File Id: " + fileId +\ ", Disk ID: " + diskId +\ ", File Version: " + str(fileVersion)]) raise Exception, errMsg # If Disk ID, File ID and File Version are given, execute a quick cloning. try: fileVersion = int(fileVersion) except: pass if (False and diskId and fileId and (fileVersion > 0)): _cloneExplicit(srvObj, reqPropsObj, diskId, fileId, fileVersion, targetDiskId) logger.info("Successfully handled command CLONE") return # Handling cloning of more files. cloneListDbm = None cloneListDbmName = tmpFilePat + "_CLONE_INFO_DB" try: # Get information about candidate files for cloning. files = srvObj.db.getFileInfoFromFileId(fileId, fileVersion, diskId, ignore=0, order=0, dbCursor=False) if not files: msg = genLog("NGAMS_ER_CMD_EXEC", [NGAMS_CLONE_CMD, "No files for cloning found"]) raise Exception(msg) # Convert to tuple of file info object plus extra info # This is how the code expects this information to come, so we need to # keep the format unless we change the bulk of the code # f[-2] is the host id, f[-1] is the mount point all_info = [] for f in files: all_info.append((ngamsFileInfo.ngamsFileInfo().unpackSqlResult(f), f[-2], f[-1])) # Create a BSD DB with information about files to be cloned. rmFile(cloneListDbmName + "*") cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName, cleanUpOnDestr=0, writePerm=1) cloneDbCount = 0 if fileId != "" and (diskId != "" or fileVersion == -1): # Take only the first element cloneListDbm.add("0", all_info[0]) cloneDbCount = 1 else: # Take all the files. for tmpFileInfo in all_info: cloneListDbm.add(str(cloneDbCount), tmpFileInfo) cloneDbCount += 1 except Exception: if (cloneListDbm): del cloneListDbm rmFile(cloneListDbmName + "*") raise logger.debug("Found: %d file(s) for cloning ...", cloneDbCount) del cloneListDbm # Check available amount of disk space. cloneCheckDiskSpace(srvObj, cloneListDbmName, tmpFilePat, targetDiskId) # Initialize Request Status parameters. if (reqPropsObj): reqPropsObj.\ setCompletionPercent(0, 1).\ setExpectedCount(cloneDbCount, 1).\ setActualCount(0, 1) srvObj.updateRequestDb(reqPropsObj) # Wait until CLONE Command has finished, or send a reply before cloning? async = 'async' in reqPropsObj and int(reqPropsObj['async']) if async: # Send intermediate reply if the HTTP Reference object is given # whenever send an auto reply now. logger.debug("CLONE command accepted - generating immediate " + "confimation reply to CLONE command") status = srvObj.genStatus(NGAMS_SUCCESS, "Accepted CLONE command for execution").\ setReqStatFromReqPropsObj(reqPropsObj).\ setActualCount(0) # Do the actual cloning in a thread args = (srvObj, cloneListDbmName, tmpFilePat, targetDiskId, reqPropsObj, None) thrName = NGAMS_CLONE_THR + threading.current_thread().getName() cloneThread = threading.Thread(None, _cloneThread, thrName, args) cloneThread.setDaemon(0) cloneThread.start() else: # Carry out the cloning (directly in this thread) and send reply # when this is done. _cloneExec(srvObj, cloneListDbmName, tmpFilePat, targetDiskId, reqPropsObj) msg = "Successfully handled command CLONE" logger.debug(msg) status = srvObj.genStatus(NGAMS_SUCCESS, msg).\ setReqStatFromReqPropsObj(reqPropsObj).setActualCount(0) rmFile(cloneListDbmName + "*") # Send reply if possible. if (httpRef): xmlStat = status.genXmlDoc(0, 0, 0, 1, 0) xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat) httpRef.send_data(xmlStat, NGAMS_XML_MT)
def _cloneExplicit(srvObj, reqPropsObj, diskId, fileId, fileVersion, targetDiskId): """ Execute CLONE Command, where the source Disk ID, File ID and File Version are specified. Is much faster than a normal CLONE Command when an explicit file is specified. srvObj: Reference to instance of Server Object (ngamsServer). fileInfoObj: File info object with info about the file diskId: ID of disk hosting the file to be cloned (string). fileId: ID of file to clone (string). fileVersion: Version of file to clone (integer). targetDiskId: ID of target disk (string). Returns: Void. """ T = TRACE(1) # Resolve the location of the file to clone. location, hostId, ipAddress, portNo, mountPoint, filename,\ fileVersion, mimeType =\ ngamsFileUtils.quickFileLocate(srvObj, reqPropsObj, fileId, diskId=diskId, fileVersion=fileVersion) # Read also the entire file info (unfortunately). srcFileInfo = ngamsFileInfo.ngamsFileInfo().read(srvObj.getHostId(), srvObj.getDb(), fileId, fileVersion, diskId) # Determine target disk. if (targetDiskId == ""): # Try to find a disk not hosting already a file with that # ID + version. diskExemptList = [diskId] while (1): trgDiskInfo = ngamsDiskUtils.\ findTargetDisk(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), mimeType, 1, diskExemptList) # Check if a file with that ID + version is already # stored on the selected Target Disk. if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fileId, fileVersion)): # This file is already stored on the given disk. # Add to the exempt list. diskExemptList.append(trgDiskInfo.getDiskId()) else: # OK, this disk should be OK, stop looking for a # suitable Target Disk. break else: trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\ read(srvObj.getDb(), targetDiskId) slotId = trgDiskInfo.getSlotId() storageSetId = srvObj.getCfg().getStorageSetFromSlotId(slotId).\ getStorageSetId() trgDiskInfo.setStorageSetId(storageSetId) # Don't accept to clone onto the same disk (this would meann overwriting). if (trgDiskInfo.getDiskId() == diskId): err = "Source and target files are identical" msg = "Failed in cloning file with ID: " + fileId +\ "/Version: " + str(fileVersion) +\ " on disk with ID: " + diskId +\ " on host: " + hostId + ". Reason: " + err raise Exception, msg # Receive the file into the staging filename. tmpReqPropsObj = ngamsReqProps.ngamsReqProps() tmpReqPropsObj.setMimeType(mimeType) stagingFilename = ngamsHighLevelLib.genStagingFilename( srvObj.getCfg(), tmpReqPropsObj, trgDiskInfo, fileId) try: quickLocation = False if (reqPropsObj.hasHttpPar("quick")): quickLocation = int(reqPropsObj.getHttpPar("quick")) # Receive the data into the Staging File using the urllib. if (srvObj.getHostId() != hostId): # Example: http://host:7777/RETRIEVE?disk_id=%s&" # file_id=id&file_version=1 fileUrl = "http://%s:%s/RETRIEVE?disk_id=%s&file_id=%s&" +\ "file_version=%s" fileUrl = fileUrl % (ipAddress, str(portNo), diskId, fileId, str(fileVersion)) # If CLONE?quick specified, we try to retrieve the file via the # RETRIEVE?quick_location method. quickFileUrl = fileUrl if (reqPropsObj.hasHttpPar("quick")): if (int(reqPropsObj.getHttpPar("quick"))): quickFileUrl = fileUrl + "&quick_location=1" # Check if host is suspended, if yes, wake it up. if (srvObj.getDb().getSrvSuspended(hostId)): logger.debug("Clone Request - Waking up suspended " +\ "NGAS Host: %s", hostId) ngamsSrvUtils.wakeUpHost(srvObj, hostId) else: # TODO: a time-bomb waiting to explode.... fileUrl = "file:" + mtPt + "/" + filename logger.debug("Receiving file via URI: %s into staging filename: %s", fileUrl, stagingFilename) # We try up to 5 times to retrieve the file in case a problem is # encountered during cloning. for attempt in range(5): try: if (attempt == 0): filename, headers = urllib.urlretrieve( quickFileUrl, stagingFilename) else: filename, headers = urllib.urlretrieve( fileUrl, stagingFilename) _checkFile(srvObj, srcFileInfo, stagingFilename, headers, True) # If we get to this point the transfer was (probably) OK. break except Exception, e: rmFile(stagingFilename) errMsg = "Problem occurred while cloning file "+\ "via URL: " + fileUrl + " - Error: " + str(e) if (attempt < 4): errMsg += " - Retrying in 5s ..." logger.error(errMsg) time.sleep(0.5) else: raise Exception, errMsg # We simply copy the file into the same destination as the # source file (but on another disk). targPathName = os.path.dirname(srcFileInfo.getFilename()) targFilename = os.path.basename(srcFileInfo.getFilename()) complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\ "/" + targPathName) checkCreatePath(complTargPath) complFilename = os.path.normpath(complTargPath + "/" + targFilename) mvTime = mvFile(stagingFilename, complFilename) ngamsLib.makeFileReadOnly(complFilename) # Update status for new file in the DB. newFileInfo = srcFileInfo.clone().setDiskId(trgDiskInfo.getDiskId()).\ setCreationDate(getFileCreationTime(complFilename)) fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fileId, fileVersion) newFileInfo.write(srvObj.getHostId(), srvObj.getDb()) # Update status for the Target Disk in DB + check if the disk is # completed. if (fileExists): mvTime = 0 dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\ setDiskId(trgDiskInfo.getDiskId()).\ setFileExists(fileExists).\ setFileSize(srcFileInfo.getFileSize()).\ setIoTime(mvTime) ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj) ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId()) # If running as a cache archive, update the Cache New Files DBM # with the information about the new file. if (srvObj.getCachingActive()): ngamsCacheControlThread.addEntryNewFilesDbm( srvObj, diskId, fileId, fileVersion, filename) # Generate a confirmation log entry. msg = genLog("NGAMS_INFO_FILE_CLONED", [fileId, fileVersion, diskId, hostId]) logger.info(msg, extra={'to_syslog': True})
def checkDbChangeCache(srvObj, diskId, diskMtPt, stopEvt): """ The function merges the information in the DB Change Snapshot Documents in the DB cache area on the disk concerned, into the Main DB Snapshot Document in a safe way which prevents that any information is lost. srvObj: Reference to NG/AMS server class object (ngamsServer). diskId: ID for disk (string). diskMtPt: Mount point of the disk, e.g. '/NGAS/disk1' (string). Returns: Void. """ if (not srvObj.getCfg().getDbSnapshot()): return if (not _updateSnapshot(srvObj.getCfg())): return snapshotDbm = None try: snapshotDbm = _openDbSnapshot(srvObj.getCfg(), diskMtPt) if (snapshotDbm == None): return # Remove possible, old /<mt pt>/.db/cache/*.xml snapshots. # TODO: Remove when it can be assumed that all old XML snapshots have # been removed. rmFile(os.path.normpath(diskMtPt + "/" + NGAMS_DB_CH_CACHE + "/*.xml")) # Update the Status document with the possibly new entries. # TODO: Potential memory bottleneck. Use 'find > file' as for # REGISTER Command. dbCacheFilePat = os.path.normpath("%s/%s/*.%s" %\ (diskMtPt, NGAMS_DB_CH_CACHE, NGAMS_PICKLE_FILE_EXT)) # Sort files by their creation date, to ensure we apply # the DB changes in the order they were generated tmpCacheFiles = glob.glob(dbCacheFilePat) tmpCacheFiles.sort(key=lambda x: os.stat(x).st_ctime) cacheStatObj = None count = 0 fileCount = 0 noOfCacheFiles = len(tmpCacheFiles) start = time.time() for cacheFile in tmpCacheFiles: checkStopJanitorThread(stopEvt) if os.lstat(cacheFile)[6] == 0: os.remove( cacheFile) # sometimes there are pickle files with 0 size. # we don't want to stop on them continue cacheStatObj = ngamsLib.loadObjPickleFile(cacheFile) if isinstance(cacheStatObj, list): # A list type in the Temporary DB Snapshot means that the # file has been removed. cacheStatList = cacheStatObj tmpFileInfoObjList = [ngamsFileInfo.ngamsFileInfo().\ setDiskId(cacheStatList[0]).\ setFileId(cacheStatList[1]).\ setFileVersion(cacheStatList[2])] operation = NGAMS_DB_CH_FILE_DELETE elif (isinstance(cacheStatObj, ngamsFileInfo.ngamsFileInfo)): tmpFileInfoObjList = [cacheStatObj] operation = cacheStatObj.getTag() else: # Assume a ngamsFileList object. cacheFileListObj = cacheStatObj.getFileListList()[0] tmpFileInfoObjList = cacheFileListObj.getFileInfoObjList() operation = cacheFileListObj.getComment() # Loop over the files in the temporary snapshot. for tmpFileInfoObj in tmpFileInfoObjList: fileKey = ngamsDbm._ensure_binary(_genFileKey(tmpFileInfoObj)) fileInfoList = tmpFileInfoObj.genSqlResult() encFileInfoDic = _encFileInfo(srvObj.getDb(), snapshotDbm, fileInfoList) if ((operation == NGAMS_DB_CH_FILE_INSERT) or (operation == NGAMS_DB_CH_FILE_UPDATE)): _addInDbm(snapshotDbm, fileKey, encFileInfoDic) tmpFileInfoObj.write(srvObj.getHostId(), srvObj.getDb(), 0) elif (operation == NGAMS_DB_CH_FILE_DELETE): if (snapshotDbm.has_key(fileKey)): del snapshotDbm[fileKey] _delFileEntry(srvObj.getHostId(), srvObj.getDb(), tmpFileInfoObj) else: # Should not happen. pass del cacheStatObj # Sleep if not last iteration (or if only one file). fileCount += 1 if (fileCount < noOfCacheFiles): time.sleep(0.010) # Synchronize the DB. count += 1 if (count == 100): snapshotDbm.sync() checkStopJanitorThread(stopEvt) count = 0 # Clean up, delete the temporary File Remove Status Document. snapshotDbm.sync() for cacheFile in tmpCacheFiles: rmFile(cacheFile) totTime = time.time() - start tmpMsg = "Handled DB Snapshot Cache Files. Mount point: %s. " +\ "Number of Cache Files handled: %d." args = [diskMtPt, fileCount] if (fileCount): tmpMsg += "Total time: %.3fs. Time per file: %.3fs." args += (totTime, (totTime / fileCount)) logger.debug(tmpMsg, *args) finally: if snapshotDbm: snapshotDbm.close()
def test_FileVerHandling_01(self): """ Synopsis: Test that REMFILE removes proper File Version. Description: The purpose of the test is to verify that if several versions of a file are found in the archive, the proper version is deleted. Expected Result: The chosen version of the file, should be selected and should be removed. Test Steps: - Start server. - Archive a file 3 times into the archive. - Clone the file onto another disk. - Remove version 2 of the cloned file. Remarks: ... Test Data: ... """ _, dbObj = self.prepExtSrv(cfgProps=(('NgamsCfg.Db[1].Snapshot', "0"), )) client = sendPclCmd() for n in range(3): stat = client.archive("src/SmallFile.fits") self.assertEquals('SUCCESS', stat.getStatus()) diskId1 = "tmp-ngamsTest-NGAS-FitsStorage1-Main-1" fileId = "TEST.2001-05-08T15:25:00.123" fileVer = 2 status = client.get_status(NGAMS_CLONE_CMD, pars=[["disk_id", diskId1]]) self.assertEquals('SUCCESS', status.getStatus()) diskId2 = "tmp-ngamsTest-NGAS-FitsStorage2-Main-3" filePath = "/tmp/ngamsTest/NGAS/FitsStorage2-Main-3/saf/" +\ "2001-05-08/2/TEST.2001-05-08T15:25:00.123.fits.gz" for execute in [0, 1]: httpPars = [["disk_id", diskId2], ["file_id", fileId], ["file_version", fileVer], ["execute", execute]] stat = client.get_status(NGAMS_REMFILE_CMD, pars=httpPars) self.assertStatus(stat) fileInfo = ngamsFileInfo.ngamsFileInfo() try: fileInfo.read(getHostName(), dbObj, fileId, fileVer, diskId2) except: pass if (execute == 0): if (not os.path.exists(filePath)): self.fail("File removed unexpectedly") else: if (os.path.exists(filePath)): self.fail("File not removed as expected") refStatFile = "ref/ngamsRemFileCmdTest_test_" +\ "FileVerHandling_01_0%d_ref" % (execute + 3) tmpStatFile = saveInFile(None, filterDbStatus1(fileInfo.dumpBuf())) self.checkFilesEq( refStatFile, tmpStatFile, "Incorrect status for REMFILE Command/execution")
def _clone(srvObj, diskId, fileId, fileVersion, targetDiskId, reqPropsObj, httpRef, tmpFilePat): """ Internal function used by ngamsCloneCmd.clone() to carry out the cloning. See documentation for ngamsCloneCmd.clone(). """ targetDiskId = targetDiskId.strip() logger.debug("Handling file cloning with parameters - File ID: %s -" + \ "Disk ID: %s - File Version: %s - Target Disk ID: |%s|", fileId, diskId, str(fileVersion), targetDiskId) if not fileId and not diskId: errMsg = genLog("NGAMS_ER_CMD_SYNTAX", [NGAMS_CLONE_CMD, "File Id: " + fileId +\ ", Disk ID: " + diskId +\ ", File Version: " + str(fileVersion)]) raise Exception(errMsg) # Get candidate files for cloning. # We convert to tuple of file info object plus extra info because # this is how the code expects this information to come, so we need to # keep the format unless we change the bulk of the code # f[-2] is the host id, f[-1] is the mount point all_info = [(ngamsFileInfo.ngamsFileInfo().unpackSqlResult(f), f[-2], f[-1]) for f in srvObj.db.getFileInfoFromFileId( fileId, fileVersion, diskId, ignore=0, order=0, order2=1, dbCursor=False)] if not all_info: msg = genLog("NGAMS_ER_CMD_EXEC", [NGAMS_CLONE_CMD, "No files for cloning found"]) raise Exception(msg) # Take only the first element in these cases if fileId != "" and (diskId != "" or fileVersion == -1): all_info = [all_info[0]] cloneDbCount = len(all_info) cloneListDbmName = tmpFilePat + "_CLONE_INFO_DB" rmFile(cloneListDbmName + "*") try: ngamsDbm.enumerate_to_dbm(cloneListDbmName, all_info) except Exception: rmFile(cloneListDbmName + "*") raise logger.debug("Found: %d file(s) for cloning ...", cloneDbCount) # Check available amount of disk space. cloneCheckDiskSpace(srvObj, cloneListDbmName, tmpFilePat, targetDiskId) # Initialize Request Status parameters. if (reqPropsObj): reqPropsObj.\ setCompletionPercent(0, 1).\ setExpectedCount(cloneDbCount, 1).\ setActualCount(0, 1) srvObj.updateRequestDb(reqPropsObj) # Wait until CLONE Command has finished, or send a reply before cloning? is_async = 'async' in reqPropsObj and int(reqPropsObj['async']) if is_async: # Send intermediate reply if the HTTP Reference object is given # whenever send an auto reply now. logger.debug("CLONE command accepted - generating immediate " + "confimation reply to CLONE command") status = srvObj.genStatus(NGAMS_SUCCESS, "Accepted CLONE command for execution").\ setReqStatFromReqPropsObj(reqPropsObj).\ setActualCount(0) # Do the actual cloning in a thread args = (srvObj, cloneListDbmName, tmpFilePat, targetDiskId, reqPropsObj, None) thrName = NGAMS_CLONE_THR + threading.current_thread().getName() cloneThread = threading.Thread(None, _cloneThread, thrName, args) cloneThread.daemon = False cloneThread.start() else: # Carry out the cloning (directly in this thread) and send reply # when this is done. _cloneExec(srvObj, cloneListDbmName, tmpFilePat, targetDiskId, reqPropsObj) msg = "Successfully handled command CLONE" logger.debug(msg) status = srvObj.genStatus(NGAMS_SUCCESS, msg).\ setReqStatFromReqPropsObj(reqPropsObj).setActualCount(0) rmFile(cloneListDbmName + "*") # Send reply if possible. if (httpRef): xmlStat = status.genXmlDoc(0, 0, 0, 1, 0) xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat) httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT)
def ingestExtFiles(fileListFile, notifEmail): """ Ingest the list of files given in the file referred to. The files are listed as: <Disk ID> <File ID> [<File Version>] ... fileListFile: Filename of file containing the file list (string). notifEmail: List of email addresses to inform about the execution of the discation procedure (string). Returns: Void. """ fileInfoList = parseFileList(fileListFile) # Open DB connection. server, db, user, password = ngasUtilsLib.getDbPars() dbCon = ngamsDb.ngamsDb(server, db, user, password, 0) # Find distinct Disk IDs. diskIdDic = {} for fileInfo in fileInfoList: diskIdDic[fileInfo[0]] = None info(1, "Disk IDs: " + str(diskIdDic.keys())) # Register disks referred to as external disks in the DB. dbDiskDic = {} for diskId in diskIdDic.keys(): if (not dbDiskDic.has_key(diskId)): diskInfo = dbCon.getDiskInfoFromDiskId(diskId) if (diskInfo != []): diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\ unpackSqlResult(diskInfo) else: # Create a new entry for that disk in the DB. info(1,"Creating new entry for external disk with ID: " +\ str(diskId)) diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\ setArchive("EXTERNAL").\ setDiskId(diskId).\ setLogicalName(diskId).\ setHostId("").\ setSlotId("").\ setMounted(0).\ setMountPoint("EXTERNAL").\ setNumberOfFiles(0).\ setAvailableMb(0).\ setBytesStored(0).\ setCompleted(1).\ setCompletionDateFromSecs(0).\ setType("EXTERNAL").\ setManufacturer("UNKNOWN").\ setInstallationDateFromSecs(time.time()).\ setChecksum(0).\ setTotalDiskWriteTime(0).\ setLastCheckFromSecs(0).\ setLastHostId("").\ setStorageSetId(diskId) diskInfoObj.write(dbCon) diskIdDic[diskId] = diskInfoObj # Loop over the files and register them in the DB. fileRegList = [] sys.stdout.write("Registering files ...") fileCount = 0 for fileInfo in fileInfoList: diskId = fileInfo[0] fileId = fileInfo[1] fileVer = fileInfo[2] fileInfoObj = ngamsFileInfo.ngamsFileInfo().\ setDiskId(diskId).\ setFilename(fileId).\ setFileId(fileId).\ setFileVersion(fileVer).\ setFormat("").\ setFileSize(0).\ setUncompressedFileSize(0).\ setCompression("").\ setIngestionDateFromSecs(0).\ setIgnore(0).\ setChecksum("").\ setChecksumPlugIn("").\ setFileStatus(NGAMS_FILE_STATUS_OK).\ setCreationDateFromSecs(0).\ setTag("EXTERNAL") fileInfoObj.write(getHostName(), dbCon, 0, 1) fileRegList.append((diskId, fileId, fileVer)) time.sleep(0.050) fileCount += 1 if ((fileCount % 10) == 0): sys.stdout.write(".") sys.stdout.write("\n") info(1, "Registered %d files" % fileCount) report = genReport(fileRegList) if (notifEmail): ngasUtilsLib.sendEmail("ngasRegisterExtFiles: FILE REGISTRATION " +\ "REPORT", notifEmail, report)
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle STATUS command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ status = ngamsStatus.ngamsStatus() status.setDate(toiso8601()).\ setVersion(getNgamsVersion()).setHostId(srvObj.getHostId()).\ setStatus(NGAMS_SUCCESS).\ setMessage("Successfully handled command STATUS").\ setState(srvObj.getState()).setSubState(srvObj.getSubState()) reqPropsObjRef = reqPropsObj # Get the information requested. diskId = "" fileId = "" fileVersion = "" configurationFile = "" fileAccess = "" hostId = "" requestId = "" dbTime = "" dbTimeReset = "" fileList = "" fileListId = "" maxElements = 100000 if (reqPropsObj.hasHttpPar("disk_id")): diskId = reqPropsObj.getHttpPar("disk_id") if (reqPropsObj.hasHttpPar("file_id")): fileId = reqPropsObj.getHttpPar("file_id") if (reqPropsObj.hasHttpPar("file_version")): fileVersion = int(reqPropsObj.getHttpPar("file_version")) if (reqPropsObj.hasHttpPar("configuration_file")): configurationFile = reqPropsObj.getHttpPar("configuration_file") if (reqPropsObj.hasHttpPar("file_access")): fileAccess = reqPropsObj.getHttpPar("file_access") if (reqPropsObj.hasHttpPar("host_id")): hostId = reqPropsObj.getHttpPar("host_id") if (reqPropsObj.hasHttpPar("max_elements")): maxElements = int(reqPropsObj.getHttpPar("max_elements")) if (reqPropsObj.hasHttpPar("request_id")): requestId = reqPropsObj.getHttpPar("request_id") if (reqPropsObj.hasHttpPar("db_time")): dbTime = True if (reqPropsObj.hasHttpPar("db_time_reset")): dbTimeReset = True if (reqPropsObj.hasHttpPar("flush_log")): # in the past this called flushLog() # do we really want to keep that functionality? I doubt it pass if (reqPropsObj.hasHttpPar("file_list")): fileList = int(reqPropsObj.getHttpPar("file_list")) if (reqPropsObj.hasHttpPar("file_list_id")): fileListId = reqPropsObj.getHttpPar("file_list_id") if (reqPropsObj.hasHttpPar("dump_object_info")): # Dump status of all objects allocated into file specified. targFile = reqPropsObj.getHttpPar("dump_object_info") rmFile(targFile) fo = open(targFile, "w") fo.write(_genObjectStatus()) fo.close() # Handle request for file info. genCfgStatus = 0 genDiskStatus = 0 genFileStatus = 0 genStatesStatus = 1 genRequestStatus = 0 msg = "" help = 0 if (reqPropsObj.hasHttpPar("help")): global _help msg = _help help = 1 elif hostId and hostId != srvObj.getHostId(): host, port = srvObj.get_remote_server_endpoint(hostId) cfgObj = srvObj.getCfg() if not cfgObj.getProxyMode(): httpRef.redirect(host, port) return else: try: httpRef.proxy_request(hostId, host, port) except Exception as e: ex = re.sub("<|>", "", str(e)) errMsg = genLog("NGAMS_ER_COM", [host, port,ex]) raise Exception(errMsg) return elif (fileList): if (not fileListId): # It's a new STATUS?file_list request. fileListId = _handleFileList(srvObj, reqPropsObj, httpRef) # Send back data from the request. _handleFileListReply(srvObj, reqPropsObj, httpRef, fileListId, maxElements) elif (diskId): diskObj = ngamsDiskInfo.ngamsDiskInfo() diskObj.read(srvObj.getDb(), diskId) status.addDiskStatus(diskObj) genDiskStatus = 1 elif (fileId): if (not fileVersion): fileVersion = -1 try: fileObj = ngamsFileInfo.ngamsFileInfo() fileObj.read(srvObj.getHostId(), srvObj.getDb(), fileId, fileVersion) diskObj = ngamsDiskInfo.ngamsDiskInfo() try: diskObj.read(srvObj.getDb(), fileObj.getDiskId()) except: errMsg = "Illegal Disk ID found: {0} for file with ID: {1}".format(fileObj.getDiskId(), fileId) raise Exception(errMsg) diskObj.addFileObj(fileObj) status.addDiskStatus(diskObj) except: # The file was not found in the database. Check if it is available # on a remote partner site. host, port, status_info, disk_info, file_info = \ ngamsFileUtils.lookup_partner_site_file_status(srvObj, fileId, fileVersion, reqPropsObj) # Update status reply using the disk status and file status # information retrieved from the partner site status.addDiskStatus(disk_info) genDiskStatus = 1 genFileStatus = 1 elif (requestId): logger.debug("Checking status of request with ID: %s", requestId) reqPropsObjRef = srvObj.getRequest(requestId) if (not reqPropsObjRef): errMsg = genLog("NGAMS_ER_ILL_REQ_ID", [requestId]) raise Exception(errMsg) genRequestStatus = 1 elif (configurationFile): msg = "configuration_file=" + srvObj.cfg_fname genCfgStatus = 1 # Hidden feature to return complete Cfg!! try: genCfgStatus = int(configurationFile) except ValueError: genCfgStatus = 1 status.setNgamsCfgObj(srvObj.getCfg()) elif (fileAccess): if (not fileVersion): fileVersion = -1 fileId = fileAccess msg = _checkFileAccess(srvObj, reqPropsObj, httpRef, fileId, fileVersion, diskId) elif (dbTime): logger.debug("Querying total DB time") msg = "Total DB time: %.6fs" % srvObj.getDb().getDbTime() elif (dbTimeReset): msg = "Resetting DB timer" logger.debug(msg) srvObj.getDb().resetDbTime() else: msg = "Successfully handled command STATUS" if (reqPropsObjRef == reqPropsObj): reqPropsObj.setCompletionTime() srvObj.updateRequestDb(reqPropsObj) if (genCfgStatus or genDiskStatus or genFileStatus or genRequestStatus): status.setReqStatFromReqPropsObj(reqPropsObjRef) # Generate XML reply. logger.debug("Status requests: " + str([genCfgStatus, genDiskStatus, genFileStatus, genStatesStatus])) xmlStat = status.genXmlDoc(genCfgStatus, genDiskStatus, genFileStatus, genStatesStatus) xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat) httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT) elif not httpRef.reply_sent: httpRef.send_status(msg) if (msg and (not help)): logger.info(msg) else: logger.info("Successfully handled command STATUS")
def _registerExec(srvObj, fileListDbmName, tmpFilePat, diskInfoDic, reqPropsObj = None): """ Register the files listed in the File List DBM (ngamsDbm), which match the mime-type(s) either specified in the 'mimeType' parameter, or if this is not specified, which match all the mime-types specified in the configuration file. When the registration procedure has been executed, the function sends an Email Notification message indicating which files were registered if the HTTP parameter 'notif_email' is given. The functions creates a File Info Objects per file handled and writes this in a temporary file, which is a DBM file. The keys in this DB is simply the file number in the sequence of files handled, pointing to a pickled ngamsFileInfo object. Each of the File Info Objects indicates if the file was registered or not. This is done by setting the tag of the File Info Object to one of the following values: REGISTERED: The file was successfully registered in the NGAS DB FAILED: The file was selected for registration but could not be properly registered because of inconsistencies. The status will be of the format: 'FAILED[: <reason>]'. REJECTED: A file found under the specified path directory was not accepted for cloning, usually because the mime-type was not correct. The status will be of the format: 'REJECTED[: <reason>]'. Note, that registration is comparable to archiving of files. For that reason a DAPI must be provided for each type of file that should be registered. If this is not fullfilled, the file registration will fail. Only files stored on one of the NGAS disks configured in the configuration file, are considered. Files stored in other locations are ignored. srvObj: Instance of NG/AMS Server object (ngamsServer). fileListDbmName: Name of a DBM containing the information about the files to be registered. Each element in the list is referred to by a key, which is a number. These points to a pickled list for each file containing the following information: [<Filename>, <Disk ID>, <Mime-Type>] The information for each disk concerned is also contained in the DB referred to by its Disk ID and by its mount point. The data is a pickled instance of the ngamsDiskInfo class (string). tmpFilePat: Pattern for temporary files used during the registration process (string). diskInfoDic: Dictionary with Disk IDs as keys pointing to the info about the disk (dictionary/ngamsDiskInfo). reqPropsObj: If an NG/AMS Request Properties Object is given, the Request Status will be updated as the request is carried out (ngamsReqProps). Returns: Void. """ T = TRACE() emailNotif = 0 if (reqPropsObj): if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1 # Create the temporary BSD DB to contain the information for the # Email Notification Message. if (emailNotif): regDbmName = tmpFilePat + "_NOTIF_EMAIL" regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm = 1) # Open the DBM containing the list of files to (possibly) register. fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm = 1) # Want to parse files in alphabetical order. # TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting. tmpFileList = tmpFilePat + "_FILE_LIST" rmFile(tmpFileList) fo = open(tmpFileList, "w") fileListDbm.initKeyPtr() while (1): dbmKey, fileInfo = fileListDbm.getNext() if (not dbmKey): break fo.write(dbmKey + "\n") fo.close() sortFileList = tmpFilePat + "_SORT_FILE_LIST" rmFile(sortFileList) shellCmd = "sort %s > %s" % (tmpFileList, sortFileList) stat, out = commands.getstatusoutput(shellCmd) if (stat != 0): raise Exception, "Error executing command: %s. Error: %s" %\ (shellCmd, str(out)) rmFile(tmpFileList) # Go through each file in the list, check if the mime-type is among the # ones, which apply for registration. If yes try to register the file # by invoking the corresponding DAPI on the file. checksumPlugIn = srvObj.getCfg().getChecksumPlugIn().strip() fileRegCount = 0 fileFailCount = 0 fileRejectCount = 0 regTimeAccu = 0.0 fileCount = 0 fo = open(sortFileList) run = 1 while (run): reg_start = time.time() dbmKey = fo.readline() if (dbmKey.strip() == ""): run = 0 continue fileInfo = fileListDbm.get(dbmKey[0:-1]) filename = fileInfo[0] diskId = fileInfo[1] mimeType = fileInfo[2] # Register the file. Check first, that exactly this file is # not already registered. In case it is, the file will be rejected. regPi = srvObj.getCfg().register_plugins[mimeType] logger.debug("Plugin found for %s: %s", mimeType, regPi) params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars) tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\ setMimeType(mimeType).\ setStagingFilename(filename).\ setTargDiskInfo(diskInfoDic[diskId]).\ setHttpMethod(NGAMS_HTTP_GET).\ setCmd(NGAMS_REGISTER_CMD).\ setSize(os.path.getsize(filename)).\ setFileUri(filename).\ setNoReplication(1) tmpFileObj = ngamsFileInfo.ngamsFileInfo() try: # Invoke Registration Plug-In. piName = regPi.name plugInMethod = loadPlugInEntryPoint(piName) piRes = plugInMethod(srvObj, tmpReqPropsObj, params) del tmpReqPropsObj # Check if this file is already registered on this disk. In case # yes, it is not registered again. files = srvObj.db.getFileSummary1(srvObj.getHostId(), [piRes.getDiskId()], [piRes.getFileId()]) fileRegistered = 0 for tmpFileInfo in files: tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT] tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME] tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\ tmpFilename) if (tmpComplFilename == filename): fileRegistered = 1 break if (fileRegistered): fileRejectCount += 1 tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\ "and path: %s is already registered on disk " +\ "with Disk ID: %s" tmpMsg = tmpMsgForm % (piRes.getFileId(), piRes.getFileVersion(), filename, piRes.getDiskId()) logger.warning(tmpMsg + ". File is not registered again.") if (emailNotif): tmpFileObj.\ setDiskId(diskId).setFilename(filename).\ setTag(tmpMsg) regDbm.addIncKey(tmpFileObj) if (reqPropsObj): reqPropsObj.incActualCount(1) ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj, regTimeAccu) regTimeAccu += time.time() - reg_start fileCount += 1 continue # Calculate checksum (if plug-in specified). if (checksumPlugIn != ""): logger.debug("Invoking Checksum Plug-In: %s to handle file: %s", checksumPlugIn, filename) plugInMethod = loadPlugInEntryPoint(checksumPlugIn) checksum = plugInMethod(srvObj, filename, 0) else: checksum = "" # Move file and update information about file in the NGAS DB. mvFile(filename, piRes.getCompleteFilename()) ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum, checksumPlugIn) ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes) ngamsLib.makeFileReadOnly(piRes.getCompleteFilename()) if (emailNotif): uncomprSize = piRes.getUncomprSize() ingestDate = time.time() creDateSecs = getFileCreationTime(filename) tmpFileObj.\ setDiskId(diskId).\ setFilename(filename).\ setFileId(piRes.getFileId()).\ setFileVersion(piRes.getFileVersion()).\ setFormat(piRes.getFormat()).\ setFileSize(piRes.getFileSize()).\ setUncompressedFileSize(uncomprSize).\ setCompression(piRes.getCompression()).\ setIngestionDate(ingestDate).\ setIgnore(0).\ setChecksum(checksum).\ setChecksumPlugIn(checksumPlugIn).\ setFileStatus(NGAMS_FILE_STATUS_OK).\ setCreationDate(creDateSecs).\ setTag("REGISTERED") fileRegCount += 1 # If running as a cache archive, update the Cache New Files DBM # with the information about the new file. if (srvObj.getCachingActive()): fileVer = fio.getFileVersion() ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId, piRes.getFileId(), fileVer, filename) # Generate a confirmation log entry. msg = genLog("NGAMS_INFO_FILE_REGISTERED", [filename, piRes.getFileId(), piRes.getFileVersion(), piRes.getFormat()]) time.sleep(0.005) regTime = time.time() - reg_start msg = msg + ". Time: %.3fs." % (regTime) logger.info(msg, extra={'to_syslog': 1}) except Exception, e: errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)]) logger.error(errMsg) if (emailNotif): tmpFileObj.\ setDiskId(diskId).setFilename(filename).\ setTag(errMsg) fileFailCount += 1 regTime = time.time() - reg_start # TODO (rtobar, 2016-01): Why don't we raise an exception here? # Otherwise the command appears as successful on the # client-side # Add the file information in the registration report. if (emailNotif): regDbm.addIncKey(tmpFileObj) # Update request status time information. regTimeAccu += regTime if (reqPropsObj): reqPropsObj.incActualCount(1) ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj, regTimeAccu) fileCount += 1
def receiveData(srvObj, reqPropsObj, httpRef): """ Receive the data in connection with the Rearchive Request. For a description of the parameters: Check handleCmd(). Returns: Tuple with File Info Object for the file to be rearchived and Disk Info Object for the selected target disk (tuple/(fileInfo, ngamsDiskInfo)). """ # Note, this algorithm does not implement support for back-log buffering # for speed optimization reasons. # Unpack the file information contained in the HTTP header # (NGAS-File-Info). encFileInfo = reqPropsObj.getHttpHdr(NGAMS_HTTP_HDR_FILE_INFO) if (not encFileInfo): msg = "Error. Must provide NGAS File Information " +\ "(RFC 3548 encoded) in HTTP header NGAS-File-Info" raise Exception(msg) fileInfoXml = base64.b64decode(encFileInfo) fileInfoObj = ngamsFileInfo.ngamsFileInfo().unpackXmlDoc(fileInfoXml) # Find the most suitable target Disk Set. trgDiskInfoObj = None diskExList = [] while (True): try: tmpTrgDiskInfo = ngamsDiskUtils.\ findTargetDisk(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), reqPropsObj.getMimeType(), sendNotification = 0, diskExemptList = diskExList, reqSpace = reqPropsObj.getSize()) if (srvObj.getDb().fileInDb(tmpTrgDiskInfo.getDiskId(), fileInfoObj.getFileId(), fileInfoObj.getFileVersion())): diskExList.append(tmpTrgDiskInfo.getDiskId()) else: trgDiskInfoObj = tmpTrgDiskInfo break except Exception as e: msg = "Error locating target disk for REARCHIVE Command. Error: %s" msg = msg % str(e) raise Exception(msg) # Generate Staging Filename + save file into this. storageSetId = trgDiskInfoObj.getStorageSetId() stagingFilename = ngamsHighLevelLib.\ genStagingFilename(srvObj.getCfg(), reqPropsObj, srvObj.getDiskDic(), storageSetId, reqPropsObj.getFileUri(), genTmpFiles = 0) reqPropsObj.setStagingFilename(stagingFilename) # If it is an Rearchive Pull Request, open the URL. if (reqPropsObj.getHttpMethod() == NGAMS_HTTP_GET): # urllib.urlopen will attempt to get the content-length based on the URI # i.e. file, ftp, http handle = urllib.urlopen(reqPropsObj.getFileUri()) reqPropsObj.setSize(handle.info()['Content-Length']) rfile = handle else: reqPropsObj.setSize(fileInfoObj.getFileSize()) rfile = httpRef.rfile # Save the data into the Staging File. try: # Make mutual exclusion on disk access (if requested). ngamsHighLevelLib.acquireDiskResource(srvObj.getCfg(), trgDiskInfoObj.getSlotId()) # If provided with a checksum, calculate the checksum on the incoming # data stream and check that it is the same as the expected one # (as indicated in the file info structure given by the user) stored_checksum = fileInfoObj.getChecksum() crc_variant = fileInfoObj.getChecksumPlugIn() skip_crc = True if stored_checksum and crc_variant: reqPropsObj.addHttpPar('crc_variant', crc_variant) reqPropsObj.__httpHdrDic[NGAMS_HTTP_HDR_CHECKSUM] = stored_checksum skip_crc = False ngamsArchiveUtils.archive_contents_from_request(stagingFilename, srvObj.getCfg(), reqPropsObj, rfile, skip_crc=skip_crc) finally: ngamsHighLevelLib.releaseDiskResource(srvObj.getCfg(), trgDiskInfoObj.getSlotId()) # Synchronize the file caches to ensure the files have been stored # on the disk and check that the files are accessible. # This sync is only relevant if back-log buffering is on. if (srvObj.getCfg().getBackLogBuffering()): ngamsFileUtils.syncCachesCheckFiles(srvObj, [stagingFilename]) return (fileInfoObj, trgDiskInfoObj)
return elif (fileList): if (not fileListId): # It's a new STATUS?file_list request. fileListId = _handleFileList(srvObj, reqPropsObj, httpRef) # Send back data from the request. _handleFileListReply(srvObj, reqPropsObj, httpRef, fileListId, maxElements) elif (diskId): diskObj = ngamsDiskInfo.ngamsDiskInfo() diskObj.read(srvObj.getDb(), diskId) status.addDiskStatus(diskObj) genDiskStatus = 1 elif (fileId): if (not fileVersion): fileVersion = -1 fileObj = ngamsFileInfo.ngamsFileInfo() fileObj.read(srvObj.getHostId(), srvObj.getDb(), fileId, fileVersion) diskObj = ngamsDiskInfo.ngamsDiskInfo() try: diskObj.read(srvObj.getDb(), fileObj.getDiskId()) except: errMsg = "Illegal Disk ID found: %s for file with ID: %s" %\ (fileObj.getDiskId(), fileId) raise Exception, errMsg diskObj.addFileObj(fileObj) status.addDiskStatus(diskObj) genDiskStatus = 1 genFileStatus = 1 elif (requestId): logger.debug("Checking status of request with ID: %s", requestId) reqPropsObjRef = srvObj.getRequest(requestId)
def _handleFileListReply(srvObj, reqPropsObj, httpRef, fileListId, maxElements = None): """ Extracts file information from a previously dumped file information in connection with a STATUS?file_list request. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). fileListId: File List ID allocated to this request (string). maxElements: Maximum number of elements to extract and return to the requestor (integer). Returns: Void. """ # Get the name of the DBM in which the information is stored. dbmBaseName = STATUS_FILE_LIST_DBM_TAG % fileListId dbmPat = os.path.normpath("%s/*%s*" %\ (ngamsHighLevelLib.\ getNgasTmpDir(srvObj.getCfg()), dbmBaseName)) dbmMatches = glob.glob(dbmPat) if (len(dbmMatches) < 1): msg = "Referenced File List ID: %s in connection with " +\ "STATUS/file_list request, is not (or no longer) known" raise Exception(msg % fileListId) elif (len(dbmMatches) > 1): msg = "Inconsistencies encountered in locating result set for " +\ "STATUS/file_list for referenced File List ID: %s" raise Exception(msg % fileListId) fileInfoDbmName = dbmMatches[0] # Generate the NG/AMS Status Document, with a File List in it. # Compress it on the fly. fileListXmlDoc = ngamsHighLevelLib.\ genTmpFilename(srvObj.getCfg(), "STATUS_FILE_LIST_XML.xml") try: tmpFileListDbm = ngamsDbm.ngamsDbm(fileInfoDbmName, cleanUpOnDestr = 0, writePerm = 1) rmFile(fileListXmlDoc) fo = open(fileListXmlDoc, "w") if (not maxElements): remainingObjects = 0 else: remainingObjects = (tmpFileListDbm.getCount() - maxElements) if (remainingObjects < 0): remainingObjects = 0 fo.write(_fileListXmlHdr % (fileListId, str(remainingObjects))) # Loop over the file info objects and write them into the file. # take max the number of elements specified. tmpFileListDbm.initKeyPtr() elCount = 0 keyRefList = [] while (True): # Have the requested number of elements been extracted? if (maxElements): if (elCount >= maxElements): break # Get the next key (if there are more elements). key, fileInfo = tmpFileListDbm.getNext() if (not key): break try: # Write the File Status XML Element in the file. tmpFileInfoObj = ngamsFileInfo.ngamsFileInfo().\ unpackSqlResult(fileInfo) fileInfoXml = tmpFileInfoObj.genXml(storeDiskId = 1).\ toprettyxml(" ", "\n")[:-1] fo.write("\n" + fileInfoXml) except Exception as e: msg = "Error creating STATUS/File List XML Document. " +\ "Error: %s" % str(e) logger.error(msg) raise keyRefList.append(key) elCount += 1 # Finish up the XML document, close the file. fo.write(_fileListXmlFooter) fo.close() # Assume this type of file can always be compressed. fileListXmlDoc = compressFile(fileListXmlDoc) except: rmFile("%s*" % fileInfoDbmName) rmFile("%s*" % fileListXmlDoc) raise # Send the XML document back to the requestor. try: httpRef.send_file(fileListXmlDoc, NGAMS_GZIP_XML_MT) # Remove the reported entries. for key in keyRefList: tmpFileListDbm.rem(key) tmpFileListDbm.sync() del keyRefList keyRefList = [] # If there are no more entries, delete the DBM. dbmCount = tmpFileListDbm.getCount() del tmpFileListDbm if (dbmCount == 0): rmFile("%s*" % fileInfoDbmName) except Exception as e: msg = "Error returning response to STATUS?file_list request. Error: %s" msg = msg % str(e) raise Exception(msg) finally: rmFile(fileListXmlDoc)
fileInfoList[ngamsDb.NGAS_FILES_CHECKSUM_PI] = checksumPlugIn fileInfoList[ngamsDb.NGAS_FILES_FILE_STATUS] = "00000000" fileInfoList[ngamsDb.NGAS_FILES_CREATION_DATE] = ingestionTime updateDbSnapshot(dbSnapshotDbm, fileInfoList) del fileInfoList # Update DB if requested. if (dbCon): fileInfoObj = ngamsFileInfo.ngamsFileInfo().\ setChecksum(checksum).\ setChecksumPlugIn(checksumPlugIn).\ setCompression(compression).\ setCreationDate(ingestionTime).\ setDiskId(diskId).\ setFileId(fileId).\ setFileSize(archFileSize).\ setFileStatus("00000000").\ setFileVersion(fileVer).\ setFilename(relFilename).\ setFormat(format).\ setIgnore(0).\ setIngestionDate(ingestionTime).\ setUncompressedFileSize(uncomprSize) fileInfoObj.write(getHostName(), dbCon, genSnapshot=0) del fileInfoObj dayFileCount += 1 fileCount += 1 # Give a signal of life ... if ((fileCount % 100) == 0):
def checkFileCopiesAndReg(srvObj, minReqCopies, dbFilePat, fileListDbmName=None, diskId=None, ignoreMounted=0): """ The function checks for each file referenced if there are at least 'minReqCopies' copies available somewhere in this NGAS cluster. For the files where this is not the case, an entry is added in a ngasDiskInfo object indicating that this file If an entire disk is analysed (about to be deleted), it is also checked if each file stored on the disk is registered in the DB. Otherwise an NG/AMS File List is returned, containing references to the files not registered. srvObj: Instance of the NG/AMS Server Class (ngamsServer). minReqCopies: Minimum number of copies required (integer). dbFilePat: Filename pattern used to build the DBM containing information about the files (string). fileListDbmName: Name of DBM DB containing explicit references to files to be checked if they can be deleted. The information in this table is pickled lists with the lay-out defined by ngamsDb._ngasFilesCols (string). diskId: Used to refer to all files stored on a disk (string|None). ignoreMounted: Carry out the check also if the disk is not mounted (integer/0|1). Returns: Tuple contaning the filenames of three DBM DBs with the following information: o Files not having the specified number of copies. The contents of this DB are keys (Disk ID + File ID + File Version), pointing to pickled ngamsFileInfo objects. o Files found on the disk but not registered. This contains the complete filenames of files found on the disk, which are not registered. These filenames are the keys of this DBM DB. o Complete list of files referenced in connection with the query. The contents of this DB are keys, which are a simple counter pointing to pickled list containing the information as returned by ngamsDb.getFileSummary1(). (tuple/string). """ T = TRACE() if ((not fileListDbmName) and (not diskId)): errMsg = "ngamsSrvUtils.checkFileCopiesAndReg(): Must specify " +\ "either a DBM with files to be checked or a Disk ID" logger.warning(errMsg) raise Exception(errMsg) # Create DBMs: # - DB containing information about files having less then the # specified number of copies. fileMisCopyDbmName = os.path.normpath(dbFilePat + "_MISSING_COPIES") fileMisCopyDbm = ngamsDbm.ngamsDbm(fileMisCopyDbmName, writePerm=1) # - DB that contains information about files stored on the DB, # which are not registered in the NGAS DB. At the end of the function, # this will contain information about files found on the disk but # not registered in the NGAS DB. filesOnDiskDicDbmName = os.path.normpath(dbFilePat + "_FILES_ON_DISK") filesOnDiskDicDbm = ngamsDbm.ngamsDbm(filesOnDiskDicDbmName, writePerm=1) # - DB with information about files referenced by the query. if (not fileListDbmName): locFileListDbmName = os.path.normpath(dbFilePat + "_FILE_LIST") fileListDbm = ngamsDbm.ngamsDbm(locFileListDbmName, writePerm=1) else: fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=0) # - Temporary DBM containing information about all File IDs defined # by the query. fileIdDbmName = os.path.normpath(dbFilePat + "_FILE_IDS") fileIdDbm = ngamsDbm.ngamsDbm(fileIdDbmName, writePerm=1) # - Temporary DBM containing information about all files available in # the system with the File ID/File Version defined by the query. complFileListDbmName = os.path.normpath(dbFilePat + "_COMPL_FILE_LIST") complFileListDbm = ngamsDbm.ngamsDbm(complFileListDbmName, writePerm=1) # - Temporary DBM that is used to figure out the number of independent # copies of each file concerned by the query. checkDicDbmName = os.path.normpath(dbFilePat + "_CHECK_DIC") checkDicDbm = ngamsDbm.ngamsDbm(checkDicDbmName, writePerm=1) # A Disk ID but no file references are given. Retrieve information # about files concerned from the DB. if (diskId): logger.debug("Retrieving information about files on disk with ID: %s", diskId) for f in srvObj.db.getFileSummary1(None, [diskId], [], ignore=0, fileStatus=[]): fileListDbm.addIncKey(f) fileId = f[ngamsDbCore.SUM1_FILE_ID] fileIdDbm.add(str(fileId), "") fileVersion = f[ngamsDbCore.SUM1_VERSION] fileKey = ngamsLib.genFileKey(None, fileId, fileVersion) checkDicDbm.add(fileKey, {}) fileListDbm.sync() fileIdDbm.sync() checkDicDbm.sync() # Get the list of files located on the disk. Later on, remove entries # from this dictionary as the files are parsed, based on their DB info, # further down in this method. # # Key in this dictionary is the complete filename of the file. logger.debug("Get list of files stored on disk ...") tmpDiskInfo = srvObj.getDb().getDiskInfoFromDiskId(diskId) diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\ unpackSqlResult(tmpDiskInfo) if ((not ignoreMounted) and (not diskInfoObj.getMounted())): errMsg = "Rejecting request for removing disk with ID: " +\ diskId + " - disk not mounted!" raise Exception(errMsg) if (not ignoreMounted): basePath = os.path.normpath(diskInfoObj.getMountPoint()) pattern = "/*" logger.debug( "Generating list with files on disk with base path: %s", basePath) while (1): tmpFileList = glob.glob(basePath + pattern) if (len(tmpFileList) == 0): break else: for filename in tmpFileList: if (os.path.isfile(filename) and (os.path.basename(filename) != NGAMS_DISK_INFO) and (os.path.basename(filename) != NGAMS_VOLUME_ID_FILE) and (os.path.basename(filename) != NGAMS_VOLUME_INFO_FILE)): filesOnDiskDicDbm.add(str(filename), "") pattern += "/*" # Generate File ID DBM in case a file list DBM is given. if (fileListDbmName): logger.debug("Handling file list DBM given in the function call ...") fileListDbm.initKeyPtr() while (1): key, tmpFileInfo = fileListDbm.getNext() if (not key): break # Update the File ID DBM. fileId = tmpFileInfo[ngamsDbCore.SUM1_FILE_ID] fileIdDbm.add(str(fileId), "") # Update the DBM with references to File ID/Version sets. fileVersion = tmpFileInfo[ngamsDbCore.SUM1_VERSION] fileKey = ngamsLib.genFileKey(None, fileId, fileVersion) checkDicDbm.add(fileKey, {}) fileIdDbm.sync() checkDicDbm.sync() # We need to generate a list with all files available in the system # with the given File ID/File Version. logger.debug("Retrieving information about all files available with the " +\ "File ID/File Version as defined by the query") # Due to the limitation of the size of SQL queries, we have to split up # the SQL query in several sub-queries. The max. length of an SQL query # is defined by NGAMS_MAX_SQL_QUERY_SZ, we subtract 512 from this for # the general part of the query, and for each filename we calculate a # length of len(File ID) + 4 as contribution to the SQL query. maxQuerySize = (NGAMS_MAX_SQL_QUERY_SZ - 512) queryIds = [] querySize = 0 noOfFileIds = fileIdDbm.getCount() fileIdCount = 0 fileIdDbm.initKeyPtr() fileId = "INIT" while (fileId): fileId, dummy = fileIdDbm.getNext() if (fileId): queryIds.append(utils.b2s(fileId)) fileIdCount += 1 querySize += (len(fileId) + 4) if queryIds and (querySize >= maxQuerySize or fileIdCount == noOfFileIds): for f in srvObj.db.getFileSummary1(fileIds=queryIds, fileStatus=[]): # Take only a sub-result if that File ID + Version # is concerned by the query. tmpFileId = f[ngamsDbCore.SUM1_FILE_ID] tmpFileVersion = f[ngamsDbCore.SUM1_VERSION] tmpFileKey = ngamsLib.genFileKey(None, tmpFileId, tmpFileVersion) if (checkDicDbm.hasKey(tmpFileKey)): complFileListDbm.addIncKey(f) complFileListDbm.sync() queryIds = [] querySize = 0 # Now, go through the files found and order these such that we end up with # a Dictionary with "<File ID>_<File Version>" as keys referring # to a dictionary with the Disk IDs of the disks hosting the files as # keys, and the information for each file on that disk as a tupple. # # It is the intention to figure out how many copies we have of each file # identified by File ID + File Version stored ON DIFFERENT STORAGE MEDIAS # + on different hosts. logger.debug("Generate DBM DB with info about independent file copies ...") complFileListDbm.initKeyPtr() while (1): fileKey, fileInfo = complFileListDbm.getNext() if (not fileKey): break checkDicKey = ngamsLib.genFileKey(None, fileInfo[ngamsDbCore.SUM1_FILE_ID], fileInfo[ngamsDbCore.SUM1_VERSION]) tmpDic = checkDicDbm.get(checkDicKey) tmpDic[fileInfo[ngamsDbCore.SUM1_DISK_ID]] = fileInfo checkDicDbm.add(checkDicKey, tmpDic) # Check if there are at least minReqCopies occurrences of the files + # check that all files are registered (if a Disk ID is specified). logger.debug("Check for files with less copies than: %s", str(minReqCopies)) checkDicDbm.initKeyPtr() while (1): checkDicKey, tmpDic = checkDicDbm.getNext() if (not checkDicKey): break tmpDicKeys = list(tmpDic) noOfCopies = len(tmpDicKeys) if (noOfCopies < minReqCopies): tmpFileInfo = tmpDic[tmpDicKeys[0]] fileId = tmpFileInfo[ngamsDbCore.SUM1_FILE_ID] fileVersion = tmpFileInfo[ngamsDbCore.SUM1_VERSION] tmpFileObj = ngamsFileInfo.ngamsFileInfo().\ setFileId(fileId).\ setFileVersion(fileVersion).\ setTag("Independent copies: " + str(noOfCopies)) fileKey = ngamsLib.genFileKey(None, fileId, fileVersion) fileMisCopyDbm.add(str(fileKey), tmpFileObj) # Remove this file from the Files On Disk DBM - do this only # if a Disk ID is specified. if (diskId): if (diskId in tmpDic): fileInfo = tmpDic[diskId] filename = os.path.\ normpath(fileInfo[ngamsDbCore.SUM1_MT_PT] +\ "/" + fileInfo[ngamsDbCore.SUM1_FILENAME]) filename = str(filename) if (filesOnDiskDicDbm.hasKey(filename)): filesOnDiskDicDbm.rem(filename) # Close all DBM objects. del fileMisCopyDbm del filesOnDiskDicDbm del fileListDbm del fileIdDbm del complFileListDbm del checkDicDbm # The DBM filesOnDiskDicDbmName now contains references to files, # which are found on the disk but not registered in the DB. return (fileMisCopyDbmName, filesOnDiskDicDbmName, complFileListDbmName)
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle the CARCHIVE Command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: (fileId, filePath) tuple. """ T = TRACE() # Is this NG/AMS permitted to handle Archive Requests? logger.debug("Is this NG/AMS permitted to handle Archive Requests?") if (not srvObj.getCfg().getAllowArchiveReq()): errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"]) raise Exception(errMsg) srvObj.checkSetState("Archive Request", [NGAMS_ONLINE_STATE], [NGAMS_IDLE_SUBSTATE, NGAMS_BUSY_SUBSTATE], NGAMS_ONLINE_STATE, NGAMS_BUSY_SUBSTATE, updateDb=False) if httpRef.command != 'POST': raise Exception("Only POST allowed for CARCHIVE") # Get mime-type (try to guess if not provided as an HTTP parameter). logger.debug( "Get mime-type (try to guess if not provided as an HTTP parameter).") if (reqPropsObj.getMimeType() == ""): mimeType = ngamsHighLevelLib.\ determineMimeType(srvObj.getCfg(), reqPropsObj.getFileUri()) reqPropsObj.setMimeType(mimeType) else: mimeType = reqPropsObj.getMimeType() # Determine the target volume, ignoring the stream concept. logger.debug("Determine the target volume, ignoring the stream concept.") targDiskInfo = ngamsArchiveUtils._random_target_volume(srvObj) if (targDiskInfo == None): errMsg = "No disk volumes are available for ingesting any files." raise Exception(errMsg) reqPropsObj.setTargDiskInfo(targDiskInfo) # Generate staging filename. logger.debug("Generate staging filename from URI: %s", reqPropsObj.getFileUri()) if (reqPropsObj.getFileUri().find("file_id=") >= 0): file_id = reqPropsObj.getFileUri().split("file_id=")[1] baseName = os.path.basename(file_id) else: baseName = os.path.basename(reqPropsObj.getFileUri()) stgFilename = os.path.join("/", targDiskInfo.getMountPoint(), NGAMS_STAGING_DIR, genUniqueId() + "___" + baseName) logger.debug("Staging filename is: %s", stgFilename) reqPropsObj.setStagingFilename(stgFilename) # Retrieve file contents (from URL, archive pull, or by storing the body # of the HTTP request, archive push). stagingInfo = saveInStagingFile(srvObj.getCfg(), reqPropsObj, httpRef, stgFilename, targDiskInfo) ioTime = stagingInfo[0] rootContainer = stagingInfo[1] fileDataList = stagingInfo[2] ingestRate = stagingInfo[3] reqPropsObj.incIoTime(ioTime) createContainers(rootContainer, None, srvObj) parDic = {} ngamsGenDapi.handlePars(reqPropsObj, parDic) diskInfo = reqPropsObj.getTargDiskInfo() # Generate file information. logger.debug("Generate file information") dateDir = toiso8601(fmt=FMT_DATE_ONLY) resDapiList = [] containerSizes = {} for item in fileDataList: container = item[0] filepath = item[1] crc = item[2] containerId = str(container.getContainerId()) basename = os.path.basename(filepath) fileId = basename fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, filepath, fileId, basename, [dateDir]) complFilename, relFilename = ngamsGenDapi.checkForDblExt( complFilename, relFilename) # Keep track of the total size of the container uncomprSize = ngamsPlugInApi.getFileSize(filepath) if containerId not in containerSizes: containerSizes[containerId] = 0 containerSizes[containerId] += uncomprSize mimeType = reqPropsObj.getMimeType() compression = "NONE" archFileSize = ngamsPlugInApi.getFileSize(filepath) resDapi = ngamsPlugInApi.genDapiSuccessStat( diskInfo.getDiskId(), relFilename, fileId, fileVersion, mimeType, archFileSize, uncomprSize, compression, relPath, diskInfo.getSlotId(), fileExists, complFilename) # Move file to final destination. logger.debug("Moving file to final destination") ioTime = mvFile(filepath, resDapi.getCompleteFilename()) reqPropsObj.incIoTime(ioTime) # Get crc info checksumPlugIn = "StreamCrc32" checksum = str(crc) # Get source file version # e.g.: http://ngas03.hq.eso.org:7778/RETRIEVE?file_version=1&file_id=X90/X962a4/X1 logger.debug("Get file version") file_version = resDapi.getFileVersion() if reqPropsObj.getFileUri().count("file_version"): file_version = int( (reqPropsObj.getFileUri().split("file_version=")[1] ).split("&")[0]) # Check/generate remaining file info + update in DB. logger.debug("Creating db entry") creDate = getFileCreationTime(resDapi.getCompleteFilename()) fileInfo = ngamsFileInfo.ngamsFileInfo().\ setDiskId(resDapi.getDiskId()).\ setFilename(resDapi.getRelFilename()).\ setFileId(resDapi.getFileId()).\ setFileVersion(file_version).\ setFormat(resDapi.getFormat()).\ setFileSize(resDapi.getFileSize()).\ setUncompressedFileSize(resDapi.getUncomprSize()).\ setCompression(resDapi.getCompression()).\ setIngestionDate(time.time()).\ setChecksum(checksum).setChecksumPlugIn(checksumPlugIn).\ setFileStatus(NGAMS_FILE_STATUS_OK).\ setCreationDate(creDate).\ setIoTime(reqPropsObj.getIoTime()) fileInfo.write(srvObj.getHostId(), srvObj.getDb()) # Add the file to the container srvObj.getDb().addFileToContainer(containerId, resDapi.getFileId(), True) # Update the container sizes for contSizeInfo in containerSizes.items(): srvObj.getDb().setContainerSize(contSizeInfo[0], contSizeInfo[1]) # Inform the caching service about the new file. logger.debug("Inform the caching service about the new file.") if (srvObj.getCachingActive()): diskId = resDapi.getDiskId() fileId = resDapi.getFileId() fileVersion = file_version filename = resDapi.getRelFilename() ngamsCacheControlThread.addEntryNewFilesDbm( srvObj, diskId, fileId, fileVersion, filename) # Update disk info in NGAS Disks. logger.debug("Update disk info in NGAS Disks.") srvObj.getDb().updateDiskInfo(resDapi.getFileSize(), resDapi.getDiskId()) resDapiList.append(resDapi) # Check if the disk is completed. # We use an approximate extimate for the remaning disk space to avoid # to read the DB. logger.debug("Check available space in disk") availSpace = getDiskSpaceAvail(targDiskInfo.getMountPoint(), smart=False) if (availSpace < srvObj.getCfg().getFreeSpaceDiskChangeMb()): targDiskInfo.setCompleted(1).setCompletionDate(time.time()) targDiskInfo.write(srvObj.getDb()) # Request after-math ... srvObj.setSubState(NGAMS_IDLE_SUBSTATE) msg = "Successfully handled Archive Pull Request for data file " +\ "with URI: " + reqPropsObj.getSafeFileUri() logger.info(msg) httpRef.send_ingest_status(msg, targDiskInfo) for resDapi in resDapiList: # Trigger Subscription Thread. This is a special version for MWA, in which we simply swapped MIRRARCHIVE and QARCHIVE # [email protected] logger.debug("triggering SubscriptionThread for file %s", resDapi.getFileId()) srvObj.addSubscriptionInfo( [(resDapi.getFileId(), resDapi.getFileVersion())], []) srvObj.triggerSubscriptionThread()