def getDpIdInfo(request): """ Generate the File ID (here DP ID) for the file. filename: Name of FITS file (string). Returns: Tuple containing the value of ARCFILE, the DP ID of the file, and the JD date. The two latter deducted from the ARCFILE keyword (tuple). """ ignore_arcfile_keyword = int(request.get('ignore_arcfile', 0)) if ignore_arcfile_keyword: filename = os.path.basename(request.getFileUri()) return filename, toiso8601(fmt=FMT_DATE_ONLY) try: keyDic = getFitsKeys(request.getStagingFilename(), ["ARCFILE"]) arcFile = keyDic["ARCFILE"][0] els = arcFile.split(".") dpId = els[0] + "." + els[1] + "." + els[2] # Make sure that the files are stored according to JD # (one night is 12am -> 12am). isoTime = '.'.join(els[1:]) ts1 = fromiso8601(isoTime) ts2 = tomjd(ts1) - 0.5 dateDirName = toiso8601(frommjd(ts2), fmt=FMT_DATE_ONLY) return dpId, dateDirName except: err = "Did not find keyword ARCFILE in FITS file or ARCFILE illegal" errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE", [os.path.basename(filename), "ngamsFitsPlugIn", err]) logger.exception(errMsg) raise
def test_duplicate_subscription(self): """ Test that creating multiple subscriptions with the same ID results in different HTTP codes returned to the client """ URL = 'http://127.0.0.1:1234/path' NOW = time.time() START_DATE = toiso8601(NOW, local=True) def assert_subscription(http_status, ngams_status, url=URL, start_date=START_DATE): status = self.client.subscribe(url=url, startDate=start_date, pars=[['subscr_id', 'my-id']]) self.assertEqual(status.http_status, http_status) self.assertEqual(status.getStatus(), ngams_status) self.prepExtSrv() assert_subscription(200, 'SUCCESS') assert_subscription(201, 'SUCCESS') assert_subscription(409, 'FAILURE', url=URL + '/subpath') assert_subscription(409, 'FAILURE', start_date=toiso8601(NOW + 1, local=True))
def printStatus(stat): """ Pretty print the return status document Input: stat: an ngamsStatus document """ message = """ Status of request: Request Time: {0} Host ID: {1} Message: {2} Status: {3} State: {4} Sub-State: {5} NG/AMS Version: {6} """ req_time = "" if stat.getRequestTime() is not None: req_time = toiso8601(stat.getRequestTime()) print( message.format( req_time, stat.getHostId(), stat.getMessage(), stat.getStatus(), stat.getState(), stat.getSubState(), stat.getVersion(), ))
def dataCheckThread(srvObj, stopEvt, checksum_allow_evt, checksum_stop_evt): """ The Data Check Thread is executed to run a periodic check of the consistency of the data files contained in an NG/AMS system. The periodic check is performed only when NG/AMS is Online. srvObj: Reference to server object (ngamsServer). Returns: Void. """ minCycleTime = isoTime2Secs(srvObj.getCfg().getDataCheckMinCycle()) logger.info("Data checker thread period is %f", minCycleTime) while True: # Encapsulate this whole block to avoid that the thread dies in # case a problem occurs, like e.g. a problem with the DB connection. try: # Wait until we're sure that the Janitor Thread has executed # at least once, to ensure that the check is carried out in a # clean environment. while (not srvObj.getJanitorThreadRunCount()): suspend(stopEvt, 0.5) logger.info("Data Check Thread starting iteration ...") # Everything happens here stats = data_check_cycle(srvObj, stopEvt, checksum_allow_evt, checksum_stop_evt) lastOldestCheck = srvObj.getDb().getMinLastDiskCheck( srvObj.getHostId()) time_to_compare = lastOldestCheck or stats.time_start execTime = time.time() - time_to_compare if execTime < minCycleTime: waitTime = minCycleTime - execTime nextAbsCheckTime = int(time.time() + waitTime) srvObj.setNextDataCheckTime(nextAbsCheckTime) logger.info( "Suspending Data Checking Thread for %.3f [s]. " "Next run scheduled for %s", waitTime, toiso8601(nextAbsCheckTime)) suspend(stopEvt, waitTime) except StopDataCheckThreadException: return except Exception: errMsg = "Error occurred during execution of the Data Check Thread" logger.exception(errMsg) try: suspend(stopEvt, 1) except StopDataCheckThreadException: return # EOF
def ngams_generic(ngams_server, request_properties): """ Data Archiving Plug-In to handle archiving of SDM multipart related message files containing ALMA UIDs in the Content-Location mime parameter or any other kind of file :param ngams_server: Reference to NG/AMS Server Object (ngamsServer) :param request_properties: NG/AMS request properties object (ngamsReqProps) :return: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus) """ logger.info("Mirroring plug-in handling data for file: %s", os.path.basename(request_properties.getFileUri())) # Create staging file disk_info = request_properties.getTargDiskInfo() staging_filename = request_properties.getStagingFilename() # request_properties format: /MIRRARCHIVE?mime_type=application/x-tar&filename=... file_format = request_properties.getMimeType() if not file_format: raise Exception("mime_type parameter not specified in MIRRARCHIVE request") # Example of file URI format: # http://ngas01.org:7777/RETRIEVE?disk_id=59622720f79296473f6106c15e5c2240&host_id=ngas01:7777&quick_location=1&file_version=1&file_id=backup.2011-02-02T22:01:59.tar file_id = request_properties.fileinfo["fileId"] file_version = request_properties.fileinfo["fileVersion"] # Specific treatment depending on the mime-type if file_format.find("multipart") >= 0 or file_format.find("multialma") >= 0: logger.debug("Mirroring plug-in applying specific treatment for multipart/multialma mime file") file_id, final_filename, file_format = specific_treatment(staging_filename) else: final_filename = file_id logger.debug("Mirroring plug-in processing request for file with URI %s, file_format=%s, file_id=%s, " "file_version=%s, final_filename=%s", request_properties.getFileUri(), file_format, file_id, file_version, final_filename) try: # Compression parameters uncompressed_size = ngamsPlugInApi.getFileSize(staging_filename) compression = "" today = ngamsCore.toiso8601(fmt=ngamsCore.FMT_DATE_ONLY) relative_path, relative_filename, complete_filename, file_exists = \ generate_file_info(ngams_server.getCfg(), disk_info, staging_filename, file_version, final_filename, [today]) # Make sure the format is defined if not file_format: file_format = ngamsPlugInApi.determineMimeType(ngams_server.getCfg(), staging_filename) file_size = ngamsPlugInApi.getFileSize(staging_filename) return ngamsPlugInApi.genDapiSuccessStat(disk_info.getDiskId(), relative_filename, file_id, file_version, file_format, file_size, uncompressed_size, compression, relative_path, disk_info.getSlotId(), file_exists, complete_filename) except Exception as e: raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE", [staging_filename, PLUGIN_ID, "Problem processing file in staging area: " + str(e)]))
def ngamsZHLDapi(srvObj, reqPropsObj): """ Generic Data Archiving Plug-In to handle archiving of any file. srvObj: Reference to NG/AMS Server Object (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus). """ T = TRACE() # For now the exception handling is pretty basic: # If something goes wrong during the handling it is tried to # move the temporary file to the Bad Files Area of the disk. logger.debug("Plug-In handling data for file: %s", os.path.basename(reqPropsObj.getFileUri())) try: parDic = {} handlePars(reqPropsObj, parDic) diskInfo = reqPropsObj.getTargDiskInfo() stgFile = reqPropsObj.getStagingFilename() ext = os.path.splitext(stgFile)[1][1:] # Generate file information. logger.debug("Generate file information") dateDir = toiso8601(fmt=FMT_DATE_ONLY) fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, reqPropsObj.\ getStagingFilename(), parDic[FILE_ID], parDic[FILE_ID], [dateDir]) complFilename, relFilename = checkForDblExt(complFilename, relFilename) # Compress the file if requested. uncomprSize, archFileSize, format, compression, comprExt =\ compressFile(srvObj, reqPropsObj, parDic) if (comprExt != ""): complFilename += ".%s" % comprExt relFilename += ".%s" % comprExt logger.debug( "DAPI finished processing file - returning to host application") insertFitsRecords(srvObj, reqPropsObj, stgFile) return ngamsPlugInApi.genDapiSuccessStat( diskInfo.getDiskId(), relFilename, parDic[FILE_ID], fileVersion, format, archFileSize, uncomprSize, compression, relPath, diskInfo.getSlotId(), fileExists, complFilename) except Exception: logger.exception("Error occurred in DAPI") raise
def populate_mirroring_bookkeeping_table(diff_ngas_files_query, start_date, db_link, cluster_name, iteration, ngams_server): """ Populate mirroring book keeping table with the difference between the source and target tables. :param diff_ngas_files_query: string, Sub-Query defining the diff between ngas_files in source and target clusters :param start_date: Start date :param db_link: string, Name for the data base link hosting the target cluster :param cluster_name: string, Name of the target cluster :param iteration: string, Iteration number for next mirroring loop :param ngams_server: ngamsServer, Reference to NG/AMS server class object """ # First dump information from diff_ngas_files into book keeping table # Insert query statement sql = "insert into ngas_mirroring_bookkeeping" + db_link + " " # Fields to be filled sql += "(file_id, file_version, disk_id, host_id, " sql += "file_size, format, status, target_cluster, " sql += "source_host, ingestion_date, " sql += "iteration, checksum, checksum_plugin, source_ingestion_date)" sql += "select " # file_id: Direct from diff_ngas_files table sql += "d.file_id, " # file_version: Direct from diff_ngas_files table sql += "d.file_version, " # disk_id: Direct from diff_ngas_files table sql += "d.disk_id, " # host_id: Direct from diff_ngas_files table sql += "d.host_id, " # file_size: Direct from diff_ngas_files table sql += "d.file_size, " # format: Direct from diff_ngas_files table sql += "d.format, " # status: Must be filled with 0 in case of no-ready entry sql += "'LOCKED'," # target_host: We can temporary use the name of the target cluster rather than the target host to lock table entry sql += "'" + cluster_name + "', " # source_host: We concatenate host_id (without port) sql += "substr(d.host_id,0,instr(d.host_id || ':',':')-1) || '.' || " # With domain name and port number sql += "d.domain || ':' || d.srv_port, " # ingestion_date: We have to assign it a default value because it is part of the primary key sql += "{1}, " # iteration: In order to distinguish from different runs sql += "{2}, " # The checksum from the source node sql += "d.checksum, d.checksum_plugin, d.ingestion_date" # All this info comes from the diff table sql += " from " + diff_ngas_files_query + " d" # if rows_limit is not None and rows_limit != "None": # query += " where rownum <= " + str(rows_limit) # args are startDate, ingestionTime and iteration args = (start_date, toiso8601(fmt=FMT_DATETIME_NOMSEC) + ":000", iteration) ngams_server.getDb().query2(sql, args=args) return
def _checkContainerClosed(self, containerName, isClosed): status = self.clist(containerName) self.assertEqual(1, len(status.getContainerList())) container = status.getContainerList()[0] self.assertEqual(containerName, container.getContainerName()) # Containers are closed when they have an ingestion date self.assertEqual( isClosed, container.isClosed(), "Container's ingestion date is: '" + toiso8601(container.getIngestionDate()) + "'; expected isClosed=" + str(isClosed))
def ngamsSdmMultipart(ngams_server, request_properties): """ Data Archiving Plug-In to handle archiving of SDM multipart related message files containing ALMA UIDs in the Content-Location mime parameter :param ngams_server: Reference to NG/AMS Server Object (ngamsServer) :param request_properties: NG/AMS request properties object (ngamsReqProps) :return: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus) """ # For now the exception handling is pretty basic: # If something goes wrong during the handling it is tried to move the temporary file to the bad-files directory logger.info("SDM multipart plug-in handling data for file: %s", os.path.basename(request_properties.getFileUri())) disk_info = request_properties.getTargDiskInfo() staging_filename = request_properties.getStagingFilename() file_id, final_filename, file_format = specific_treatment(staging_filename) if request_properties.hasHttpPar("file_id"): file_id = request_properties.getHttpPar("file_id") #if request_properties.hasHttpPar("file_version"): # file_version = request_properties.getHttpPar("file_version") logger.debug("SDM multipart plug-in processing request for file with URI %s, file_format=%s, file_id=%s, " "final_filename=%s", request_properties.getFileUri(), file_format, file_id, final_filename) try: # Compression parameters uncompressed_size = ngamsPlugInApi.getFileSize(staging_filename) compression = "" # Remember to update the temporary file name in the request properties object request_properties.setStagingFilename(staging_filename) today = ngamsCore.toiso8601(fmt=ngamsCore.FMT_DATE_ONLY) file_version, relative_path, relative_filename, complete_filename, file_exists = \ ngamsPlugInApi.genFileInfo(ngams_server.getDb(), ngams_server.getCfg(), request_properties, disk_info, staging_filename, file_id, final_filename, [today]) # Make sure the format is defined if not file_format: file_format = ngamsPlugInApi.determineMimeType(ngams_server.getCfg(), staging_filename) file_size = ngamsPlugInApi.getFileSize(staging_filename) return ngamsPlugInApi.genDapiSuccessStat(disk_info.getDiskId(), relative_filename, file_id, file_version, file_format, file_size, uncompressed_size, compression, relative_path, disk_info.getSlotId(), file_exists, complete_filename) except Exception as e: raise Exception(genLog("NGAMS_ER_DAPI_BAD_FILE", [staging_filename, PLUGIN_ID, "Problem processing file in staging area: " + str(e)]))
def ngasTarBallPlugIn(srvObj, reqPropsObj): """ Data Archiving Plug-In to handle archiving of tarballs. srvObj: Reference to NG/AMS Server Object (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus). """ stagingFilename = reqPropsObj.getStagingFilename() logger.info("Plug-In handling data for file with URI: %s", os.path.basename(reqPropsObj.getFileUri())) diskInfo = reqPropsObj.getTargDiskInfo() # Check file. checkTarball(stagingFilename) # Get various information about the file being handled. fileId = os.path.basename(reqPropsObj.getFileUri()) timestamp = fileId[fileId.find(".") + 1:] obsDay = tomjd(fromiso8601(timestamp)) - 0.5 dateDirName = toiso8601(obsDay, fmt=FMT_DATE_ONLY) baseFilename = fileId[0:-4] fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, stagingFilename, fileId, baseFilename, [dateDirName], []) # Generate status. logger.debug("Generating status ...") format = "application/x-tar" fileSize = ngamsPlugInApi.getFileSize(stagingFilename) logger.debug("DAPI finished processing of file") return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename, fileId, fileVersion, format, fileSize, fileSize, "NONE", relPath, diskInfo.getSlotId(), fileExists, complFilename)
def test_filelist(self): """Checks that the STATUS command handles the file_list option correctly""" self.prepExtSrv() start = toiso8601() def run_checks(): fname = genTmpFilename(suffix='.xml.gz') self.status(output=fname, pars=(('file_list', 1), )) self.status(output=fname, pars=(('file_list', 1), ('from_ingestion_date', start))) self.status(output=fname, pars=(('file_list', 1), ('from_ingestion_date', start), ('unique', 1))) # Checks should be scucessfull with and wihtout files archived run_checks() self.archive('src/SmallFile.fits', 'application/octet-stream') run_checks()
def test_filelist(self): """Checks that the STATUS command handles the file_list option correctly""" self.prepExtSrv() client = sendPclCmd() start = toiso8601() def run_checks(): self.assertStatus( client.status(output='tmp/list.xml.gz', pars=(('file_list', 1), ))) self.assertStatus( client.status(output='tmp/list.xml.gz', pars=(('file_list', 1), ('from_ingestion_date', start)))) self.assertStatus( client.status(output='tmp/list.xml.gz', pars=(('file_list', 1), ('from_ingestion_date', start), ('unique', 1)))) # Checks should be scucessfull with and wihtout files archived run_checks() self.assertArchive('src/SmallFile.fits', 'application/octet-stream') run_checks()
def _genReport(srvObj, unregistered, diskDic, dbmObjDic, stats): """ Generate the DCC Check Report according to the problems found. srvObj: Reference to instance of ngamsServer object (ngamsServer). Returns: Void. """ # Find out how many inconsistencies were found. noOfProbs = 0 # Errors found. for diskId in diskDic.keys(): noOfProbs += dbmObjDic[diskId][1].getCount() # Spurious files on disk. unRegFiles = len(unregistered) # Generate the report. checkTime = time.time() - stats.time_start if ((noOfProbs + unRegFiles) or srvObj.getCfg().getDataCheckForceNotif()): report = "" hdrForm = "%-20s %s\n" format = "%-60s %-32s %-9s %s\n" separator = 130 * "-" + "\n" # Build up the report. report = "DATA CHECKING REPORT:\n\n" report += hdrForm % ("Date", toiso8601()) report += hdrForm % ("NGAS Host ID", srvObj.getHostId()) report += hdrForm % ("Start Time", toiso8601(stats.time_start)) report += hdrForm % ("Total Time (s)", "%.3f" % checkTime) report += hdrForm % ("Total Time (hours)", "%.3f" % (checkTime / 3600)) report += hdrForm % ("Rate (MB/s)", "%.3f" % stats.check_rate) report += hdrForm % ("Files Checked", stats.files_checked) report += hdrForm % ("Data Checked (MB)", "%.5f" % stats.mbs_checked) report += hdrForm % ("Inconsistencies", str(noOfProbs + unRegFiles)) report += separator # Any discrepancies found? if ((not noOfProbs) and (not unRegFiles)): report += "No discrepancies found!" # Inconsistencies found? if (noOfProbs): report += "INCONSISTENT FILES FOUND:\n\n" report += format % ("Problem Description", "File ID", "Version", "Slot ID:Disk ID") report += separator for diskId in diskDic.keys(): errDbm = dbmObjDic[diskId][1].initKeyPtr() ################################################################################################# #jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath # when iterating at the end of the table that are prone to corrupt the hash table object #while (1): # fileKey, errInfo = errDbm.getNext() # if (not fileKey): break for fileKey, dbVal in errDbm.iteritems(): # jagonzal: We need to reformat the values and skip administrative elements ################# if (str(fileKey).find("__") != -1): continue errInfo = cPickle.loads(dbVal) ############################################################################################# slotDiskId = errInfo[3] + ":" + errInfo[4] report += format % (errInfo[0], errInfo[1], errInfo[2], slotDiskId) ################################################################################################# report += separator # Not registered files found? if (unRegFiles): repFormat = "%-32s %s\n" report += "NOT REGISTERED FILES FOUND ON STORAGE DISKS:\n\n" report += repFormat % ("Disk ID:", "Filename:") report += separator for filename, disk_id in unregistered.items(): report += repFormat % (disk_id, filename) report += separator # Send Notification Message if needed (only if disks where checked). if (len(diskDic.keys())): ngamsNotification.notify(srvObj.getHostId(), srvObj.getCfg(), NGAMS_NOTIF_DATA_CHECK, "DATA CHECK REPORT", report, [], 1) # Give out the statistics for the checking. msg = genLog("NGAMS_INFO_DATA_CHK_STAT", [ stats.files_checked, unRegFiles, noOfProbs, stats.mbs_checked, stats.check_rate, checkTime ]) logger.info(msg) # Remove the various DBMs allocated. for diskId in diskDic.keys(): dbmObjDic[diskId][1].cleanUp() del dbmObjDic[diskId]
def _cloneExec(srvObj, cloneListDbmName, tmpFilePat, targetDiskId, reqPropsObj): """ See documentation of ngamsCloneCmd._cloneThread(). This function is merely implemented in order to encapsulate the whole process to be able to clean up properly when the processing is terminated. """ cloneStatusDbm = None emailNotif = 0 checkChecksum = 1 if (reqPropsObj): if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1 if (reqPropsObj.hasHttpPar("check")): checkChecksum = int(reqPropsObj.getHttpPar("check")) # Open clone list DB. cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName) # We have to get the port numbers of the hosts where the files to be # cloned are stored. hostInfoDic = {} cloneListDbm.initKeyPtr() while (1): key, fileInfo = cloneListDbm.getNext() if (not key): break hostInfoDic[fileInfo[1]] = -1 hostInfoDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), hostInfoDic.keys()) # The cloning loop. Loop over the list of files to clone and generate # a report with the result. if (emailNotif): cloneStatusDbmName = tmpFilePat + "_CLONE_STATUS_DB" cloneStatusDbm = ngamsDbm.ngamsDbm(cloneStatusDbmName, cleanUpOnDestr = 0, writePerm = 1) successCloneCount = 0 failedCloneCount = 0 abortCloneLoop = 0 timeAccu = 0.0 key = 0 while (1): clone_start = time.time() if (not cloneListDbm.hasKey(str(key))): break fileInfo = cloneListDbm.get(str(key)) key += 1 # Check if we have permission to run. Otherwise, stop. if (not srvObj.run_async_commands): break fio = fileInfo[0] mtPt = fileInfo[2] if (emailNotif): tmpFileList = ngamsFileList.\ ngamsFileList("FILE_CLONE_STATUS", "File: " + fio.getFileId() + "/" +\ fio.getDiskId() + "/" +\ str(fio.getFileVersion())) hostId = fileInfo[1] text = "Cloning file - File ID: %s/%d, on disk " +\ "with ID: %s on host: %s" logger.debug(text, fio.getFileId(), fio.getFileVersion(), fio.getDiskId(), hostId) # We generate a local Staging File and archive this. stagingFilename = "" try: # Check if file is marked as bad. if (fio.getFileStatus()[0] == "1"): errMsg = "File marked as bad - skipping!" raise Exception(errMsg) if (targetDiskId == ""): # Try to find a disk not hosting already a file with that # ID + version. diskExemptList = [fio.getDiskId()] while (1): trgDiskInfo = ngamsDiskUtils.\ findTargetDisk(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), fio.getFormat(), 1, diskExemptList) # Check if a file with that ID + version is already # stored on the selected Target Disk. if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fio.getFileId(), fio.getFileVersion())): # This file is already stored on the given disk. # Add to the exempt list. diskExemptList.append(trgDiskInfo.getDiskId()) else: # OK, this disk should be OK, stop looking for a # suitable Target Disk. break else: try: trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\ read(srvObj.getDb(), targetDiskId) slotId = trgDiskInfo.getSlotId() storageSetId = srvObj.getCfg().\ getStorageSetFromSlotId(slotId).\ getStorageSetId() trgDiskInfo.setStorageSetId(storageSetId) except Exception: abortCloneLoop = 1 raise # We don't accept to clone onto the same disk (this would mean # overwriting). if (trgDiskInfo.getDiskId() == fio.getDiskId()): err = "Source and target files are identical" msg = "Failed in cloning file with ID: " + fio.getFileId() +\ "/Version: " + str(fio.getFileVersion()) +\ " on disk with ID: " + fio.getDiskId() +\ " on host: " + hostId + ". Reason: " + err logger.warning(msg) if (emailNotif): tmpFileList.setStatus(NGAMS_FAILURE + ": " + err) tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE")) cloneStatusDbm.addIncKey(tmpFileList) failedCloneCount += 1 continue tmpReqPropsObj = ngamsReqProps.ngamsReqProps() tmpReqPropsObj.setMimeType(fio.getFormat()) stagingFilename = ngamsHighLevelLib.\ genStagingFilename(srvObj.getCfg(), tmpReqPropsObj, trgDiskInfo, fio.getFileId()) # Receive the data into the Staging File using the urllib. if (srvObj.getHostId() != hostId): # Example: http://host:7777/RETRIEVE?file_id=id&file_version=1 ipAddress = hostInfoDic[hostId].getIpAddress() portNo = hostInfoDic[hostId].getSrvPort() fileUrl = "http://" + ipAddress + ":" + str(portNo) +\ "/RETRIEVE?" + "file_id=" + fio.getFileId() +\ "&file_version=" + str(fio.getFileVersion()) # If a specific Disk ID for the source file is given, append # this. if (fio.getDiskId()): fileUrl += "&disk_id=%s" % fio.getDiskId() # Check if host is suspended, if yes, wake it up. if (srvObj.getDb().getSrvSuspended(hostId)): logger.debug("Clone Request - Waking up suspended " +\ "NGAS Host: %s", hostId) ngamsSrvUtils.wakeUpHost(srvObj, hostId) else: fileUrl = "file:" + mtPt + "/" + fio.getFilename() logger.debug("Receiving file via URI: %s into staging filename: %s", fileUrl, stagingFilename) # We try up to 5 times to retrieve the file in case a problem is # encountered during cloning. for attempt in range(5): try: filename, headers = urlrequest.urlretrieve(fileUrl, stagingFilename) _checkFile(srvObj, fio, stagingFilename, headers, checkChecksum) # If we get to this point the transfer was (probably) OK. break except Exception as e: rmFile(stagingFilename) errMsg = "Problem occurred while cloning file "+\ "via URL: " + fileUrl + " - Error: " + str(e) if (attempt < 4): errMsg += " - Retrying in 5s ..." logger.error(errMsg) time.sleep(5) else: raise Exception(errMsg) # We simply copy the file into the same destination as the # source file (but on another disk). targPathName = os.path.dirname(fio.getFilename()) targFilename = os.path.basename(fio.getFilename()) complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\ "/" + targPathName) checkCreatePath(complTargPath) complFilename = os.path.normpath(complTargPath + "/"+targFilename) mvTime = mvFile(stagingFilename, complFilename) ngamsLib.makeFileReadOnly(complFilename) # Update status for new file in the DB. newFileInfo = fio.clone().setDiskId(trgDiskInfo.getDiskId()).\ setCreationDate(getFileCreationTime(complFilename)) fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fio.getFileId(), fio.getFileVersion()) newFileInfo.write(srvObj.getHostId(), srvObj.getDb()) # Update status for the Target Disk in DB + check if the disk is # completed. if (fileExists): mvTime = 0 dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\ setDiskId(trgDiskInfo.getDiskId()).\ setFileExists(fileExists).\ setFileSize(fio.getFileSize()).setIoTime(mvTime) ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj) ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId()) # Update the clone file status list. if (emailNotif): tmpFileList.setStatus(NGAMS_SUCCESS) tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE")) tmpFileList.addFileInfoObj(newFileInfo.setTag("TARGET_FILE")) cloneStatusDbm.addIncKey(tmpFileList) successCloneCount += 1 # If running as a cache archive, update the Cache New Files DBM # with the information about the new file. if (srvObj.getCachingActive()): diskId = trgDiskInfo.getDiskId() fileId = fio.getFileId() fileVer = fio.getFileVersion() filename = fio.getFilename() ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId, fileId, fileVer, filename) # Generate a confirmation log entry. cloneTime = time.time() - clone_start timeAccu += cloneTime msg = genLog("NGAMS_INFO_FILE_CLONED", [fio.getFileId(), fio.getFileVersion(), fio.getDiskId(), hostId]) msg = msg + ". Time: %.3fs. Total time: %.3fs." %\ (cloneTime, timeAccu) logger.info(msg, extra={'to_syslog': True}) except Exception as e: cloneTime = time.time() - clone_start timeAccu += cloneTime errMsg = genLog("NGAMS_ER_FILE_CLONE_FAILED", [fio.getFileId(), fio.getFileVersion(), fio.getDiskId(), hostId, str(e)]) if (abortCloneLoop): logger.error(errMsg, extra={'to_syslog': True}) return else: logger.warning(errMsg) if (emailNotif): tmpFileList.setStatus(NGAMS_FAILURE + ": Error: " + errMsg) tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE")) cloneStatusDbm.addIncKey(tmpFileList) failedCloneCount += 1 # Delete Staging File if already created. if ((stagingFilename != "") and (os.path.exists(stagingFilename))): rmFile(stagingFilename) # Calculate time statistics. if (reqPropsObj): ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj.\ incActualCount(1), timeAccu) # Final update of the Request Status. if (reqPropsObj): complPercent = (100.0 * (float(reqPropsObj.getActualCount()) / float(reqPropsObj.getExpectedCount()))) reqPropsObj.setCompletionPercent(complPercent, 1) reqPropsObj.setCompletionTime(1) srvObj.updateRequestDb(reqPropsObj) # Send Clone Report with list of files cloned to a possible # requestor(select) of this. totFiles = (successCloneCount + failedCloneCount) if (emailNotif): xmlStat = 0 # TODO: Generation of XML status report is disabled since we cannot # handle for the moment XML documents with 1000s of elements. if (xmlStat): cloneStatusFileList = ngamsFileList.\ ngamsFileList("FILE_CLONING_STATUS_REPORT", "File Cloning Status Report") fileCount = 0 while (fileCount < cloneStatusDbm.getCount()): tmpFileList = cloneStatusDbm.get(str(fileCount)) cloneStatusFileList.addFileListObj(tmpFileList) # Make overall status. cloneStatusFileList.setStatus("SUCCESS: " +\ str(successCloneCount) +\ ", FAILURE: " +\ str(failedCloneCount) +\ ", NOT DONE: " +\ str(len(cloneStatusFileList) -\ successCloneCount -\ failedCloneCount)) status = srvObj.genStatus(NGAMS_SUCCESS, "CLONE command status report").\ addFileList(cloneStatusFileList) statRep = status.genXmlDoc(0, 0, 0, 1, 0) statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep) mimeType = NGAMS_XML_MT else: # Generate a 'simple' ASCII report. statRep = tmpFilePat + "_NOTIF_EMAIL.txt" fo = open(statRep, "w") if (reqPropsObj.hasHttpPar("disk_id")): diskId = reqPropsObj.getHttpPar("disk_id") else: diskId = "-----" if (reqPropsObj.hasHttpPar("file_id")): fileId = reqPropsObj.getHttpPar("file_id") else: fileId = "-----" if (reqPropsObj.hasHttpPar("file_version")): fileVersion = reqPropsObj.getHttpPar("file_version") else: fileVersion = "-----" tmpFormat = "CLONE STATUS REPORT:\n\n" +\ "==Summary:\n\n" +\ "Date: %s\n" +\ "NGAS Host: %s\n" +\ "Disk ID: %s\n" +\ "File ID: %s\n" +\ "File Version: %s\n" +\ "Total Number of Files: %d\n" +\ "Number of Cloned Files: %d\n" +\ "Number of Failed Files: %d\n" +\ "Total processing time (s): %.3f\n" +\ "Handling time per file (s): %.3f\n\n" +\ "==File List:\n\n" fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), diskId, fileId, str(fileVersion), totFiles, successCloneCount, failedCloneCount, timeAccu, (timeAccu / totFiles))) tmpFormat = "%-70s %-70s %-7s\n" fo.write(tmpFormat % ("Source File", "Target File", "Status")) fo.write(tmpFormat % (70 * "-", 70 * "-", 7 * "-")) key = 1 while (1): if (not cloneStatusDbm.hasKey(str(key))): break tmpFileList = cloneStatusDbm.get(str(key)) key += 1 srcFileObj = tmpFileList.getFileInfoObjList()[0] srcFile = "%s/%s/%d" % (srcFileObj.getDiskId(), srcFileObj.getFileId(), srcFileObj.getFileVersion()) if (tmpFileList.getStatus() == NGAMS_SUCCESS): trgFileObj = tmpFileList.getFileInfoObjList()[1] trgFile = "%s/%s/%d" % (trgFileObj.getDiskId(), trgFileObj.getFileId(), trgFileObj.getFileVersion()) else: trgFile = "-----" fo.write(tmpFormat % (srcFile,trgFile,tmpFileList.getStatus())) fo.write(149 * "-") fo.write("\n\n==END\n") fo.close() mimeType = NGAMS_TEXT_MT # Send out the status report. emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",") attachmentName = "CloneStatusReport" if (reqPropsObj.hasHttpPar("disk_id")): attachmentName += "-" + reqPropsObj.getHttpPar("disk_id") if (reqPropsObj.hasHttpPar("file_id")): attachmentName += "-" + reqPropsObj.getHttpPar("file_id") if (reqPropsObj.hasHttpPar("file_version")): attachmentName += "-" + reqPropsObj.getHttpPar("file_version") ngamsNotification.notify(srvObj.host_id, srvObj.cfg, NGAMS_NOTIF_INFO, "CLONE STATUS REPORT", statRep, recList=emailAdrList, force=1, contentType=mimeType, attachmentName=attachmentName) del cloneStatusDbm rmFile(cloneStatusDbmName + "*") rmFile(statRep) if (cloneListDbm): del cloneListDbm rmFile(cloneListDbmName + "*") logger.info("_cloneExec(). Total time: %.3fs. Average time per file: %.3fs.", timeAccu, (timeAccu / totFiles))
def ngamsAlmaMultipart(srvObj, reqPropsObj): """ Data Archiving Plug-In to handle archiving of ALMA multipart related message files containing ALMA UIDs. srvObj: Reference to NG/AMS Server Object (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus). """ # For now the exception handling is pretty basic: # If something goes wrong during the handling it is tried to # move the temporary file to the Bad Files Area of the disk. logger.info("Plug-In handling data for file: " + os.path.basename(reqPropsObj.getFileUri())) diskInfo = reqPropsObj.getTargDiskInfo() stagingFilename = reqPropsObj.getStagingFilename() ext = os.path.splitext(stagingFilename)[1][1:] fo = open(stagingFilename, "r") (fileId, finalName, format) = specificTreatment(fo) fo.close() try: # Compress the file. uncomprSize = ngamsPlugInApi.getFileSize(stagingFilename) compression = "" # info(2,"Compressing file using: %s ..." % compression) # exitCode, stdOut = ngamsPlugInApi.execCmd("%s %s" %\ # (compression, # stagingFilename)) # if (exitCode != 0): # errMsg = _PLUGIN_ID+": Problems during archiving! " +\ # "Compressing the file failed" # raise Exception, errMsg # stagingFilename = stagingFilename + ".Z" # Remember to update the Temporary Filename in the Request # Properties Object. reqPropsObj.setStagingFilename(stagingFilename) # info(2,"File compressed") # ToDo: Handling of non-existing fileId # if (fileId == -1): # fileId = ngamsPlugInApi.genNgasId(srvObj.getCfg()) date = toiso8601(local=True, fmt=FMT_DATE_ONLY) fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, stagingFilename, fileId, finalName, [date]) # Generate status. logger.debug("Generating status ...") if not format: format = ngamsPlugInApi.determineMimeType(srvObj.getCfg(), stagingFilename) fileSize = ngamsPlugInApi.getFileSize(stagingFilename) return ngamsPlugInApi.genDapiSuccessStat(diskInfo.getDiskId(), relFilename, fileId, fileVersion, format, fileSize, uncomprSize, compression, relPath, diskInfo.getSlotId(), fileExists, complFilename) except Exception as e: errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE", [os.path.basename(stagingFilename), _PLUGIN_ID, str(e)]) raise Exception(errMsg)
fileVersion = "-----" tmpFormat = "CLONE STATUS REPORT:\n\n" +\ "==Summary:\n\n" +\ "Date: %s\n" +\ "NGAS Host: %s\n" +\ "Disk ID: %s\n" +\ "File ID: %s\n" +\ "File Version: %s\n" +\ "Total Number of Files: %d\n" +\ "Number of Cloned Files: %d\n" +\ "Number of Failed Files: %d\n" +\ "Total processing time (s): %.3f\n" +\ "Handling time per file (s): %.3f\n\n" +\ "==File List:\n\n" fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), diskId, fileId, str(fileVersion), totFiles, successCloneCount, failedCloneCount, timeAccu, (timeAccu / totFiles))) tmpFormat = "%-70s %-70s %-7s\n" fo.write(tmpFormat % ("Source File", "Target File", "Status")) fo.write(tmpFormat % (70 * "-", 70 * "-", 7 * "-")) key = 1 while (1): if (not cloneStatusDbm.hasKey(str(key))): break tmpFileList = cloneStatusDbm.get(str(key)) key += 1 srcFileObj = tmpFileList.getFileInfoObjList()[0] srcFile = "%s/%s/%d" % (srcFileObj.getDiskId(), srcFileObj.getFileId(), srcFileObj.getFileVersion()) if (tmpFileList.getStatus() == NGAMS_SUCCESS):
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle STATUS command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ status = ngamsStatus.ngamsStatus() status.setDate(toiso8601()).\ setVersion(getNgamsVersion()).setHostId(srvObj.getHostId()).\ setStatus(NGAMS_SUCCESS).\ setMessage("Successfully handled command STATUS").\ setState(srvObj.getState()).setSubState(srvObj.getSubState()) reqPropsObjRef = reqPropsObj # Get the information requested. diskId = "" fileId = "" fileVersion = "" configurationFile = "" fileAccess = "" hostId = "" requestId = "" dbTime = "" dbTimeReset = "" fileList = "" fileListId = "" maxElements = 100000 if (reqPropsObj.hasHttpPar("disk_id")): diskId = reqPropsObj.getHttpPar("disk_id") if (reqPropsObj.hasHttpPar("file_id")): fileId = reqPropsObj.getHttpPar("file_id") if (reqPropsObj.hasHttpPar("file_version")): fileVersion = int(reqPropsObj.getHttpPar("file_version")) if (reqPropsObj.hasHttpPar("configuration_file")): configurationFile = reqPropsObj.getHttpPar("configuration_file") if (reqPropsObj.hasHttpPar("file_access")): fileAccess = reqPropsObj.getHttpPar("file_access") if (reqPropsObj.hasHttpPar("host_id")): hostId = reqPropsObj.getHttpPar("host_id") if (reqPropsObj.hasHttpPar("max_elements")): maxElements = int(reqPropsObj.getHttpPar("max_elements")) if (reqPropsObj.hasHttpPar("request_id")): requestId = reqPropsObj.getHttpPar("request_id") if (reqPropsObj.hasHttpPar("db_time")): dbTime = True if (reqPropsObj.hasHttpPar("db_time_reset")): dbTimeReset = True if (reqPropsObj.hasHttpPar("flush_log")): # in the past this called flushLog() # do we really want to keep that functionality? I doubt it pass if (reqPropsObj.hasHttpPar("file_list")): fileList = int(reqPropsObj.getHttpPar("file_list")) if (reqPropsObj.hasHttpPar("file_list_id")): fileListId = reqPropsObj.getHttpPar("file_list_id") if (reqPropsObj.hasHttpPar("dump_object_info")): # Dump status of all objects allocated into file specified. targFile = reqPropsObj.getHttpPar("dump_object_info") rmFile(targFile) fo = open(targFile, "w") fo.write(_genObjectStatus()) fo.close() # Handle request for file info. genCfgStatus = 0 genDiskStatus = 0 genFileStatus = 0 genStatesStatus = 1 genRequestStatus = 0 msg = "" help = 0 if (reqPropsObj.hasHttpPar("help")): global _help msg = _help help = 1 elif hostId and hostId != srvObj.getHostId(): host, port = srvObj.get_remote_server_endpoint(hostId) cfgObj = srvObj.getCfg() if not cfgObj.getProxyMode(): httpRef.redirect(host, port) return else: try: httpRef.proxy_request(hostId, host, port) except Exception as e: ex = re.sub("<|>", "", str(e)) errMsg = genLog("NGAMS_ER_COM", [host, port,ex]) raise Exception(errMsg) return elif (fileList): if (not fileListId): # It's a new STATUS?file_list request. fileListId = _handleFileList(srvObj, reqPropsObj, httpRef) # Send back data from the request. _handleFileListReply(srvObj, reqPropsObj, httpRef, fileListId, maxElements) elif (diskId): diskObj = ngamsDiskInfo.ngamsDiskInfo() diskObj.read(srvObj.getDb(), diskId) status.addDiskStatus(diskObj) genDiskStatus = 1 elif (fileId): if (not fileVersion): fileVersion = -1 try: fileObj = ngamsFileInfo.ngamsFileInfo() fileObj.read(srvObj.getHostId(), srvObj.getDb(), fileId, fileVersion) diskObj = ngamsDiskInfo.ngamsDiskInfo() try: diskObj.read(srvObj.getDb(), fileObj.getDiskId()) except: errMsg = "Illegal Disk ID found: {0} for file with ID: {1}".format(fileObj.getDiskId(), fileId) raise Exception(errMsg) diskObj.addFileObj(fileObj) status.addDiskStatus(diskObj) except: # The file was not found in the database. Check if it is available # on a remote partner site. host, port, status_info, disk_info, file_info = \ ngamsFileUtils.lookup_partner_site_file_status(srvObj, fileId, fileVersion, reqPropsObj) # Update status reply using the disk status and file status # information retrieved from the partner site status.addDiskStatus(disk_info) genDiskStatus = 1 genFileStatus = 1 elif (requestId): logger.debug("Checking status of request with ID: %s", requestId) reqPropsObjRef = srvObj.getRequest(requestId) if (not reqPropsObjRef): errMsg = genLog("NGAMS_ER_ILL_REQ_ID", [requestId]) raise Exception(errMsg) genRequestStatus = 1 elif (configurationFile): msg = "configuration_file=" + srvObj.cfg_fname genCfgStatus = 1 # Hidden feature to return complete Cfg!! try: genCfgStatus = int(configurationFile) except ValueError: genCfgStatus = 1 status.setNgamsCfgObj(srvObj.getCfg()) elif (fileAccess): if (not fileVersion): fileVersion = -1 fileId = fileAccess msg = _checkFileAccess(srvObj, reqPropsObj, httpRef, fileId, fileVersion, diskId) elif (dbTime): logger.debug("Querying total DB time") msg = "Total DB time: %.6fs" % srvObj.getDb().getDbTime() elif (dbTimeReset): msg = "Resetting DB timer" logger.debug(msg) srvObj.getDb().resetDbTime() else: msg = "Successfully handled command STATUS" if (reqPropsObjRef == reqPropsObj): reqPropsObj.setCompletionTime() srvObj.updateRequestDb(reqPropsObj) if (genCfgStatus or genDiskStatus or genFileStatus or genRequestStatus): status.setReqStatFromReqPropsObj(reqPropsObjRef) # Generate XML reply. logger.debug("Status requests: " + str([genCfgStatus, genDiskStatus, genFileStatus, genStatesStatus])) xmlStat = status.genXmlDoc(genCfgStatus, genDiskStatus, genFileStatus, genStatesStatus) xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat) httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT) elif not httpRef.reply_sent: httpRef.send_status(msg) if (msg and (not help)): logger.info(msg) else: logger.info("Successfully handled command STATUS")
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle SUBSCRIBE Command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ T = TRACE() """ if (srvObj.getDataMoverOnlyActive() and len(srvObj.getSubscriberDic()) > 0): srvObj.reply(reqPropsObj, httpRef, NGAMS_HTTP_SUCCESS, NGAMS_FAILURE, "Data Mover NGAS server can have only one subscriber. Use command USUBSCRIBE to update an existing subscriber.") return """ priority = 10 url = "" startDate = time.time() filterPi = "" filterPiPars = "" if (reqPropsObj.hasHttpPar("priority")): priority = reqPropsObj.getHttpPar("priority") if (reqPropsObj.hasHttpPar("url")): url = reqPropsObj.getHttpPar("url") ngamsSubscriber.validate_url(url) else: errMsg = genLog("NGAMS_ER_CMD_SYNTAX", [NGAMS_SUBSCRIBE_CMD, "Missing parameter: url"]) raise Exception, errMsg if (reqPropsObj.hasHttpPar("start_date")): tmpStartDate = reqPropsObj.getHttpPar("start_date").strip() if tmpStartDate: startDate = fromiso8601(tmpStartDate, local=True) if (reqPropsObj.hasHttpPar("filter_plug_in")): filterPi = reqPropsObj.getHttpPar("filter_plug_in") if (reqPropsObj.hasHttpPar("plug_in_pars")): filterPiPars = reqPropsObj.getHttpPar("plug_in_pars") if (reqPropsObj.hasHttpPar("subscr_id")): id = reqPropsObj.getHttpPar("subscr_id") else: id = ngamsLib.getSubscriberId(url) logger.info("Creating subscription for files >= %s", toiso8601(startDate)) subscrObj = ngamsSubscriber.ngamsSubscriber(srvObj.getHostId(), srvObj.getCfg().getPortNo(), priority, url, startDate, filterPi, filterPiPars, subscrId=id) # supports concurrent file transfer, added by [email protected] if (reqPropsObj.hasHttpPar("concurrent_threads")): concurthrds = reqPropsObj.getHttpPar("concurrent_threads") subscrObj.setConcurrentThreads(concurthrds) # If the Start Date given in before the Last Ingestion Date, we # reset the Last Ingestion Date subscrStat = srvObj.getDb().getSubscriberStatus([subscrObj.getId()], subscrObj.getHostId(), subscrObj.getPortNo()) if subscrStat: lastIngDate = subscrStat[0][1] if startDate < lastIngDate: subscrObj.setLastFileIngDate(None) else: subscrObj.setLastFileIngDate(lastIngDate) # Register the Subscriber. addSubscriber(srvObj, subscrObj) # Trigger the Data Susbcription Thread to make it check if there are # files to deliver to the new Subscriber. srvObj.addSubscriptionInfo([], [subscrObj]).triggerSubscriptionThread() return "Handled SUBSCRIBE command"
timePerFile = (regTimeAccu / fileCount) else: timePerFile = 0 tmpFormat = "REGISTER STATUS REPORT:\n\n" +\ "==Summary:\n\n" +\ "Date: %s\n" +\ "NGAS Host: %s\n" +\ "Search Path: %s\n" +\ "Total Number of Files: %d\n" +\ "Number of Registered Files: %d\n" +\ "Number of Failed Files: %d\n" +\ "Number of Rejected Files: %d\n" +\ "Total processing time (s): %.3f\n" +\ "Handling time per file (s): %.3f\n\n" +\ "==File List:\n\n" fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), path, fileCount, fileRegCount, fileFailCount, fileRejectCount, regTimeAccu, timePerFile)) tmpFormat = "%-80s %-32s %-3s %-10s\n" fo.write(tmpFormat %\ ("Filename", "File ID", "Ver", "Status")) fo.write(tmpFormat % (80 * "-", 32 * "-", 3 * "-", 10 * "-")) regDbm.initKeyPtr() while (1): key, tmpFileObj = regDbm.getNext() if (not key): break mtPt = diskInfoDic[tmpFileObj.getDiskId()].getMountPoint() filename = os.path.normpath(mtPt + "/" +\ tmpFileObj.getFilename()) line = tmpFormat %\ (filename, tmpFileObj.getFileId(),
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle SUBSCRIBE Command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ """ if (srvObj.getDataMoverOnlyActive() and len(srvObj.getSubscriberDic()) > 0): srvObj.reply(reqPropsObj, httpRef, NGAMS_HTTP_SUCCESS, NGAMS_FAILURE, "Data Mover NGAS server can have only one subscriber. Use command USUBSCRIBE to update an existing subscriber.") return """ priority = 10 url = "" startDate = time.time() filterPi = "" filterPiPars = "" if (reqPropsObj.hasHttpPar("priority")): priority = reqPropsObj.getHttpPar("priority") if (reqPropsObj.hasHttpPar("url")): url = reqPropsObj.getHttpPar("url") ngamsSubscriber.validate_url(url) else: errMsg = genLog("NGAMS_ER_CMD_SYNTAX", [NGAMS_SUBSCRIBE_CMD, "Missing parameter: url"]) raise Exception(errMsg) if (reqPropsObj.hasHttpPar("start_date")): tmpStartDate = reqPropsObj.getHttpPar("start_date").strip() if tmpStartDate: startDate = fromiso8601(tmpStartDate, local=True) if (reqPropsObj.hasHttpPar("filter_plug_in")): filterPi = reqPropsObj.getHttpPar("filter_plug_in") if (reqPropsObj.hasHttpPar("plug_in_pars")): filterPiPars = reqPropsObj.getHttpPar("plug_in_pars") if (reqPropsObj.hasHttpPar("subscr_id")): id = reqPropsObj.getHttpPar("subscr_id") else: id = ngamsLib.getSubscriberId(url) logger.info("Creating subscription for files >= %s", toiso8601(startDate)) subscrObj = ngamsSubscriber.ngamsSubscriber(srvObj.getHostId(), srvObj.getCfg().getPortNo(), priority, url, startDate, filterPi, filterPiPars, subscrId=id) # supports concurrent file transfer, added by [email protected] if (reqPropsObj.hasHttpPar("concurrent_threads")): concurthrds = reqPropsObj.getHttpPar("concurrent_threads") subscrObj.setConcurrentThreads(concurthrds) # If the Start Date given in before the Last Ingestion Date, we # reset the Last Ingestion Date # # TODO (rtobar, May 2021): # This bit of logic seems to be in contention with the fact that adding # a subscription with an ID that is already taken would be otherwise a # failure. I'm leaving it here for the time being in order to avoid # breaking anything, but it must probably be removed (or clarification # should be sought otherwise). subscrStat = srvObj.getDb().getSubscriberStatus([subscrObj.getId()], subscrObj.getHostId(), subscrObj.getPortNo()) if subscrStat and subscrStat[0][1]: lastIngDate = fromiso8601(subscrStat[0][1], local=True) if startDate < lastIngDate: subscrObj.setLastFileIngDate('') else: subscrObj.setLastFileIngDate(lastIngDate) # Register the Subscriber. existence_test = addSubscriber(srvObj, subscrObj) if existence_test == 'equal': return 201, NGAMS_SUCCESS, "Identical subscription with ID '%s' existed" % ( id, ) elif existence_test == 'unequal': return 409, NGAMS_FAILURE, "Different subscription with ID '%s' existed" % ( id, ) return "Handled SUBSCRIBE command"
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle STATUS command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ T = TRACE() status = ngamsStatus.ngamsStatus() status.\ setDate(toiso8601()).\ setVersion(getNgamsVersion()).setHostId(srvObj.getHostId()).\ setStatus(NGAMS_SUCCESS).\ setMessage("Successfully handled command STATUS").\ setState(srvObj.getState()).setSubState(srvObj.getSubState()) reqPropsObjRef = reqPropsObj # Get the information requested. diskId = "" fileId = "" fileVersion = "" configurationFile = "" fileAccess = "" hostId = "" requestId = "" dbTime = "" dbTimeReset = "" fileList = "" fileListId = "" maxElements = 100000 if (reqPropsObj.hasHttpPar("disk_id")): diskId = reqPropsObj.getHttpPar("disk_id") if (reqPropsObj.hasHttpPar("file_id")): fileId = reqPropsObj.getHttpPar("file_id") if (reqPropsObj.hasHttpPar("file_version")): fileVersion = reqPropsObj.getHttpPar("file_version") if (reqPropsObj.hasHttpPar("configuration_file")): configurationFile = "-" if (reqPropsObj.hasHttpPar("file_access")): fileAccess = reqPropsObj.getHttpPar("file_access") if (reqPropsObj.hasHttpPar("host_id")): hostId = reqPropsObj.getHttpPar("host_id") if (reqPropsObj.hasHttpPar("max_elements")): maxElements = int(reqPropsObj.getHttpPar("max_elements")) if (reqPropsObj.hasHttpPar("request_id")): requestId = reqPropsObj.getHttpPar("request_id") if (reqPropsObj.hasHttpPar("db_time")): dbTime = True if (reqPropsObj.hasHttpPar("db_time_reset")): dbTimeReset = True if (reqPropsObj.hasHttpPar("flush_log")): # in the past this called flushLog() # do we really want to keep that functionality? I doubt it pass if (reqPropsObj.hasHttpPar("file_list")): fileList = int(reqPropsObj.getHttpPar("file_list")) if (reqPropsObj.hasHttpPar("file_list_id")): fileListId = reqPropsObj.getHttpPar("file_list_id") if (reqPropsObj.hasHttpPar("dump_object_info")): # Dump status of all objects allocated into file specified. targFile = reqPropsObj.getHttpPar("dump_object_info") rmFile(targFile) fo = open(targFile, "w") fo.write(_genObjectStatus()) fo.close() # Handle request for file info. genCfgStatus = 0 genDiskStatus = 0 genFileStatus = 0 genStatesStatus = 1 genRequestStatus = 0 msg = "" help = 0 if (reqPropsObj.hasHttpPar("help")): global _help msg = _help help = 1 elif hostId and hostId != srvObj.getHostId(): host, port = srvObj.get_remote_server_endpoint(hostId) cfgObj = srvObj.getCfg() if not cfgObj.getProxyMode(): httpRef.redirect(host, port) return else: try: httpRef.proxy_request(hostId, host, port) except Exception, e: ex = re.sub("<|>", "", str(e)) errMsg = genLog("NGAMS_ER_COM", [host, port,ex]) raise Exception, errMsg return
finally: if snapshotDbm: snapshotDbm.close() if tmpSnapshotDbm: tmpSnapshotDbm.close() # Check if lost files found. logger.debug("Check if there are Lost Files ...") noOfLostFiles = lostFileRefsDbm.getCount() if (noOfLostFiles): statRep = os.path.normpath(tmpDir + "/" + ngasId +\ "_LOST_FILES_NOTIF_EMAIL.txt") fo = open(statRep, "w") timeStamp = toiso8601() tmpFormat = "JANITOR THREAD - LOST FILES DETECTED:\n\n" +\ "==Summary:\n\n" +\ "Date: %s\n" +\ "NGAS Host ID: %s\n" +\ "Lost Files: %d\n\n" +\ "==File List:\n\n" fo.write(tmpFormat % (timeStamp, srvObj.getHostId(), noOfLostFiles)) tmpFormat = "%-32s %-32s %-12s %-80s\n" fo.write(tmpFormat % ("Disk ID", "File ID", "File Version", "Expected Path")) fo.write(tmpFormat % (32 * "-", 32 * "-", 12 * "-", 80 * "-")) # Loop over the files an generate the report. lostFileRefsDbm.initKeyPtr()
def checkUpdateDbSnapShots(srvObj, stopEvt): """ Check if a DB Snapshot exists for the DB connected. If not, this is created according to the contents of the NGAS DB (if possible). During this creation it is checked if the file are physically stored on the disk. srvObj: Reference to NG/AMS server class object (ngamsServer). Returns: Void. """ snapshotDbm = None tmpSnapshotDbm = None if (not srvObj.getCfg().getDbSnapshot()): logger.debug("NOTE: DB Snapshot Feature is switched off") return logger.debug("Generate list of disks to check ...") tmpDiskIdMtPtList = srvObj.getDb().getDiskIdsMtPtsMountedDisks( srvObj.getHostId()) diskIdMtPtList = [] for diskId, mtPt in tmpDiskIdMtPtList: diskIdMtPtList.append([mtPt, diskId]) diskIdMtPtList.sort() logger.debug("Generated list of disks to check: %s", str(diskIdMtPtList)) # Generate temporary snapshot filename. ngasId = srvObj.getHostId() tmpDir = ngamsHighLevelLib.getTmpDir(srvObj.getCfg()) # Temporary DBM with file info from the DB. tmpSnapshotDbmName = os.path.normpath(tmpDir + "/" + ngasId + "_" +\ NGAMS_DB_NGAS_FILES) # Temporary DBM to contain information about 'lost files', i.e. files, # which are registered in the DB and found in the DB Snapshot, but # which are not found on the disk. logger.debug("Create DBM to hold information about lost files ...") lostFileRefsDbmName = os.path.normpath(tmpDir + "/" + ngasId +\ "_LOST_FILES") rmFile(lostFileRefsDbmName + "*") lostFileRefsDbm = ngamsDbm.ngamsDbm(lostFileRefsDbmName, writePerm=1) # Carry out the check. for mtPt, diskId in diskIdMtPtList: checkStopJanitorThread(stopEvt) logger.debug("Check/create/update DB Snapshot for disk with " +\ "mount point: %s", mtPt) try: snapshotDbm = _openDbSnapshot(srvObj.getCfg(), mtPt) if (snapshotDbm == None): continue # The scheme for synchronizing the Snapshot and the DB is: # # - Loop over file entries in the Snapshot: # - If in DB: # - If file on disk -> OK, do nothing. # - If file not on disk -> Accumulate + issue collective warning. # # - If entry not in DB: # - If file on disk -> Add entry in DB. # - If file not on disk -> Remove entry from Snapshot. # # - Loop over entries for that disk in the DB: # - If entry in Snapshot -> OK, do nothing. # - If entry not in Snapshot: # - If file on disk -> Add entry in Snapshot. # - If file not on disk -> Remove entry from DB. # Create a temporary DB Snapshot with the files from the DB. try: rmFile(tmpSnapshotDbmName + "*") tmpSnapshotDbm = bsddb.hashopen(tmpSnapshotDbmName, "c") for fileInfo in srvObj.db.getFileInfoList(diskId, ignore=None): fileKey = _genFileKey(fileInfo) encFileInfoDic = _encFileInfo(srvObj.getDb(), tmpSnapshotDbm, fileInfo) _addInDbm(tmpSnapshotDbm, fileKey, encFileInfoDic) checkStopJanitorThread(stopEvt) tmpSnapshotDbm.sync() except: rmFile(tmpSnapshotDbmName) raise ##################################################################### # Loop over the possible entries in the DB Snapshot and compare # these against the DB. ##################################################################### logger.debug("Loop over file entries in the DB Snapshot - %s ...", diskId) count = 0 try: key, pickleValue = snapshotDbm.first() except Exception as e: msg = "Exception raised accessing DB Snapshot for disk: %s. " +\ "Error: %s" logger.debug(msg, diskId, str(e)) key = None snapshotDbm.dbc = None # Create a DBM which is used to keep the list of files to remove # from the DB Snapshot. snapshotDelDbmName = ngamsHighLevelLib.\ genTmpFilename(srvObj.getCfg(), NGAMS_DB_NGAS_FILES) snapshotDelDbm = ngamsDbm.ngamsDbm(snapshotDelDbmName, cleanUpOnDestr=1, writePerm=1) ################################################################################################# #jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath # when iterating at the end of the table that are prone to corrupt the hash table object #while (key): for key, pickleValue in snapshotDbm.iteritems(): ################################################################################################# value = cPickle.loads(pickleValue) # Check if an administrative element, if yes add it if necessary. if b"___" in key: if (not tmpSnapshotDbm.has_key(key)): tmpSnapshotDbm[key] = pickleValue else: tmpFileObj = _encFileInfo2Obj(srvObj.getDb(), snapshotDbm, value) if (tmpFileObj is None): continue complFilename = os.path.normpath(mtPt + "/" +\ tmpFileObj.getFilename()) # Is the file in the DB? if (tmpSnapshotDbm.has_key(key)): # Is the file on the disk? if (not os.path.exists(complFilename)): fileVer = tmpFileObj.getFileVersion() tmpFileObj.setTag(complFilename) fileKey = ngamsLib.genFileKey( tmpFileObj.getDiskId(), tmpFileObj.getFileId(), fileVer) lostFileRefsDbm.add(fileKey, tmpFileObj) lostFileRefsDbm.sync() elif (not tmpSnapshotDbm.has_key(key)): tmpFileObj = _encFileInfo2Obj(srvObj.getDb(), snapshotDbm, value) if (tmpFileObj is None): continue # Is the file on the disk? if (os.path.exists(complFilename)): # Add this entry in the NGAS DB. tmpFileObj.write(srvObj.getHostId(), srvObj.getDb(), 0, 1) tmpSnapshotDbm[key] = pickleValue else: # Remove this entry from the DB Snapshot. msg = "Scheduling entry: %s in DB Snapshot " +\ "for disk with ID: %s for removal" logger.debug(msg, diskId, key) # Add entry in the DB Snapshot Deletion DBM marking # the entry for deletion. if (_updateSnapshot(srvObj.getCfg())): snapshotDelDbm.add(key, 1) del tmpFileObj # Be friendly and sync the DB file every now and then count += 1 if (count % 100) == 0: if _updateSnapshot(srvObj.getCfg()): snapshotDbm.sync() checkStopJanitorThread(stopEvt) tmpSnapshotDbm.sync() ################################################################################################# #jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath # when iterating at the end of the table that are prone to corrupt the hash table object #try: # key, pickleValue = snapshotDbm.next() #except: # key = None # snapshotDbm.dbc = None ################################################################################################# # Now, delete entries in the DB Snapshot if there are any scheduled for # deletion. ################################################################################################# #jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath # when iterating at the end of the table that are prone to corrupt the hash table object #snapshotDelDbm.initKeyPtr() #while (True): # key, value = snapshotDelDbm.getNext() # if (not key): break for key, value in snapshotDelDbm.iteritems(): # jagonzal: We need to reformat the values and skip administrative elements ################# if b'__' in key: continue ############################################################################################# msg = "Removing entry: %s from DB Snapshot for disk with ID: %s" logger.debug(msg, key, diskId) del snapshotDbm[key] ################################################################################################# del snapshotDelDbm logger.debug("Looped over file entries in the DB Snapshot - %s", diskId) # End-Loop: Check DB against DB Snapshot. ########################### if (_updateSnapshot(srvObj.getCfg())): snapshotDbm.sync() tmpSnapshotDbm.sync() logger.info( "Checked/created/updated DB Snapshot for disk with mount point: %s", mtPt) ##################################################################### # Loop over the entries in the DB and compare these against the # DB Snapshot. ##################################################################### logger.debug("Loop over the entries in the DB - %s ...", diskId) count = 0 try: key, pickleValue = tmpSnapshotDbm.first() except: key = None tmpSnapshotDbm.dbc = None ################################################################################################# #jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath # when iterating at the end of the table that are prone to corrupt the hash table object #while (key): for key, pickleValue in tmpSnapshotDbm.iteritems(): ################################################################################################# value = cPickle.loads(pickleValue) # Check if it is an administrative element, if yes add it if needed if b"___" in key: if (not snapshotDbm.has_key(key)): snapshotDbm[key] = pickleValue else: # Is the file in the DB Snapshot? if (not snapshotDbm.has_key(key)): tmpFileObj = _encFileInfo2Obj(srvObj.getDb(), tmpSnapshotDbm, value) if (tmpFileObj is None): continue # Is the file on the disk? complFilename = os.path.normpath(mtPt + "/" +\ tmpFileObj.getFilename()) if (os.path.exists(complFilename)): # Add this entry in the DB Snapshot. if (_updateSnapshot(srvObj.getCfg())): snapshotDbm[key] = pickleValue else: # Remove this entry from the DB (if it is there). _delFileEntry(srvObj.getHostId(), srvObj.getDb(), tmpFileObj) del tmpFileObj else: # We always update the DB Snapshot to ensure it is # in-sync with the DB entry. if (_updateSnapshot(srvObj.getCfg())): snapshotDbm[key] = pickleValue # Be friendly and sync the DB file every now and then count += 1 if (count % 100) == 0: if _updateSnapshot(srvObj.getCfg()): snapshotDbm.sync() checkStopJanitorThread(stopEvt) ################################################################################################# #jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath # when iterating at the end of the table that are prone to corrupt the hash table object #try: # key, pickleValue = tmpSnapshotDbm.next() #except: # key = None ################################################################################################# logger.debug("Checked DB Snapshot against DB - %s", diskId) # End-Loop: Check DB Snapshot against DB. ########################### if (_updateSnapshot(srvObj.getCfg())): snapshotDbm.sync() finally: if snapshotDbm: snapshotDbm.close() if tmpSnapshotDbm: tmpSnapshotDbm.close() # Check if lost files found. logger.debug("Check if there are Lost Files ...") noOfLostFiles = lostFileRefsDbm.getCount() if (noOfLostFiles): statRep = os.path.normpath(tmpDir + "/" + ngasId +\ "_LOST_FILES_NOTIF_EMAIL.txt") fo = open(statRep, "w") timeStamp = toiso8601() tmpFormat = "JANITOR THREAD - LOST FILES DETECTED:\n\n" +\ "==Summary:\n\n" +\ "Date: %s\n" +\ "NGAS Host ID: %s\n" +\ "Lost Files: %d\n\n" +\ "==File List:\n\n" fo.write(tmpFormat % (timeStamp, srvObj.getHostId(), noOfLostFiles)) tmpFormat = "%-32s %-32s %-12s %-80s\n" fo.write(tmpFormat % ("Disk ID", "File ID", "File Version", "Expected Path")) fo.write(tmpFormat % (32 * "-", 32 * "-", 12 * "-", 80 * "-")) # Loop over the files an generate the report. lostFileRefsDbm.initKeyPtr() while (1): key, fileInfoObj = lostFileRefsDbm.getNext() if (not key): break diskId = fileInfoObj.getDiskId() fileId = fileInfoObj.getFileId() fileVersion = fileInfoObj.getFileVersion() filename = fileInfoObj.getTag() fo.write(tmpFormat % (diskId, fileId, fileVersion, filename)) fo.write("\n\n==END\n") fo.close() ngamsNotification.notify(srvObj.getHostId(), srvObj.getCfg(), NGAMS_NOTIF_DATA_CHECK, "LOST FILE(S) DETECTED", statRep, [], 1, NGAMS_TEXT_MT, "JANITOR_THREAD_LOST_FILES", 1) rmFile(statRep) logger.debug("Number of lost files found: %d", noOfLostFiles) # Clean up. del lostFileRefsDbm rmFile(lostFileRefsDbmName + "*")
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle the CARCHIVE Command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: (fileId, filePath) tuple. """ T = TRACE() # Is this NG/AMS permitted to handle Archive Requests? logger.debug("Is this NG/AMS permitted to handle Archive Requests?") if (not srvObj.getCfg().getAllowArchiveReq()): errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"]) raise Exception(errMsg) srvObj.checkSetState("Archive Request", [NGAMS_ONLINE_STATE], [NGAMS_IDLE_SUBSTATE, NGAMS_BUSY_SUBSTATE], NGAMS_ONLINE_STATE, NGAMS_BUSY_SUBSTATE, updateDb=False) if httpRef.command != 'POST': raise Exception("Only POST allowed for CARCHIVE") # Get mime-type (try to guess if not provided as an HTTP parameter). logger.debug( "Get mime-type (try to guess if not provided as an HTTP parameter).") if (reqPropsObj.getMimeType() == ""): mimeType = ngamsHighLevelLib.\ determineMimeType(srvObj.getCfg(), reqPropsObj.getFileUri()) reqPropsObj.setMimeType(mimeType) else: mimeType = reqPropsObj.getMimeType() # Determine the target volume, ignoring the stream concept. logger.debug("Determine the target volume, ignoring the stream concept.") targDiskInfo = ngamsArchiveUtils._random_target_volume(srvObj) if (targDiskInfo == None): errMsg = "No disk volumes are available for ingesting any files." raise Exception(errMsg) reqPropsObj.setTargDiskInfo(targDiskInfo) # Generate staging filename. logger.debug("Generate staging filename from URI: %s", reqPropsObj.getFileUri()) if (reqPropsObj.getFileUri().find("file_id=") >= 0): file_id = reqPropsObj.getFileUri().split("file_id=")[1] baseName = os.path.basename(file_id) else: baseName = os.path.basename(reqPropsObj.getFileUri()) stgFilename = os.path.join("/", targDiskInfo.getMountPoint(), NGAMS_STAGING_DIR, genUniqueId() + "___" + baseName) logger.debug("Staging filename is: %s", stgFilename) reqPropsObj.setStagingFilename(stgFilename) # Retrieve file contents (from URL, archive pull, or by storing the body # of the HTTP request, archive push). stagingInfo = saveInStagingFile(srvObj.getCfg(), reqPropsObj, httpRef, stgFilename, targDiskInfo) ioTime = stagingInfo[0] rootContainer = stagingInfo[1] fileDataList = stagingInfo[2] ingestRate = stagingInfo[3] reqPropsObj.incIoTime(ioTime) createContainers(rootContainer, None, srvObj) parDic = {} ngamsGenDapi.handlePars(reqPropsObj, parDic) diskInfo = reqPropsObj.getTargDiskInfo() # Generate file information. logger.debug("Generate file information") dateDir = toiso8601(fmt=FMT_DATE_ONLY) resDapiList = [] containerSizes = {} for item in fileDataList: container = item[0] filepath = item[1] crc = item[2] containerId = str(container.getContainerId()) basename = os.path.basename(filepath) fileId = basename fileVersion, relPath, relFilename,\ complFilename, fileExists =\ ngamsPlugInApi.genFileInfo(srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, filepath, fileId, basename, [dateDir]) complFilename, relFilename = ngamsGenDapi.checkForDblExt( complFilename, relFilename) # Keep track of the total size of the container uncomprSize = ngamsPlugInApi.getFileSize(filepath) if containerId not in containerSizes: containerSizes[containerId] = 0 containerSizes[containerId] += uncomprSize mimeType = reqPropsObj.getMimeType() compression = "NONE" archFileSize = ngamsPlugInApi.getFileSize(filepath) resDapi = ngamsPlugInApi.genDapiSuccessStat( diskInfo.getDiskId(), relFilename, fileId, fileVersion, mimeType, archFileSize, uncomprSize, compression, relPath, diskInfo.getSlotId(), fileExists, complFilename) # Move file to final destination. logger.debug("Moving file to final destination") ioTime = mvFile(filepath, resDapi.getCompleteFilename()) reqPropsObj.incIoTime(ioTime) # Get crc info checksumPlugIn = "StreamCrc32" checksum = str(crc) # Get source file version # e.g.: http://ngas03.hq.eso.org:7778/RETRIEVE?file_version=1&file_id=X90/X962a4/X1 logger.debug("Get file version") file_version = resDapi.getFileVersion() if reqPropsObj.getFileUri().count("file_version"): file_version = int( (reqPropsObj.getFileUri().split("file_version=")[1] ).split("&")[0]) # Check/generate remaining file info + update in DB. logger.debug("Creating db entry") creDate = getFileCreationTime(resDapi.getCompleteFilename()) fileInfo = ngamsFileInfo.ngamsFileInfo().\ setDiskId(resDapi.getDiskId()).\ setFilename(resDapi.getRelFilename()).\ setFileId(resDapi.getFileId()).\ setFileVersion(file_version).\ setFormat(resDapi.getFormat()).\ setFileSize(resDapi.getFileSize()).\ setUncompressedFileSize(resDapi.getUncomprSize()).\ setCompression(resDapi.getCompression()).\ setIngestionDate(time.time()).\ setChecksum(checksum).setChecksumPlugIn(checksumPlugIn).\ setFileStatus(NGAMS_FILE_STATUS_OK).\ setCreationDate(creDate).\ setIoTime(reqPropsObj.getIoTime()) fileInfo.write(srvObj.getHostId(), srvObj.getDb()) # Add the file to the container srvObj.getDb().addFileToContainer(containerId, resDapi.getFileId(), True) # Update the container sizes for contSizeInfo in containerSizes.items(): srvObj.getDb().setContainerSize(contSizeInfo[0], contSizeInfo[1]) # Inform the caching service about the new file. logger.debug("Inform the caching service about the new file.") if (srvObj.getCachingActive()): diskId = resDapi.getDiskId() fileId = resDapi.getFileId() fileVersion = file_version filename = resDapi.getRelFilename() ngamsCacheControlThread.addEntryNewFilesDbm( srvObj, diskId, fileId, fileVersion, filename) # Update disk info in NGAS Disks. logger.debug("Update disk info in NGAS Disks.") srvObj.getDb().updateDiskInfo(resDapi.getFileSize(), resDapi.getDiskId()) resDapiList.append(resDapi) # Check if the disk is completed. # We use an approximate extimate for the remaning disk space to avoid # to read the DB. logger.debug("Check available space in disk") availSpace = getDiskSpaceAvail(targDiskInfo.getMountPoint(), smart=False) if (availSpace < srvObj.getCfg().getFreeSpaceDiskChangeMb()): targDiskInfo.setCompleted(1).setCompletionDate(time.time()) targDiskInfo.write(srvObj.getDb()) # Request after-math ... srvObj.setSubState(NGAMS_IDLE_SUBSTATE) msg = "Successfully handled Archive Pull Request for data file " +\ "with URI: " + reqPropsObj.getSafeFileUri() logger.info(msg) httpRef.send_ingest_status(msg, targDiskInfo) for resDapi in resDapiList: # Trigger Subscription Thread. This is a special version for MWA, in which we simply swapped MIRRARCHIVE and QARCHIVE # [email protected] logger.debug("triggering SubscriptionThread for file %s", resDapi.getFileId()) srvObj.addSubscriptionInfo( [(resDapi.getFileId(), resDapi.getFileVersion())], []) srvObj.triggerSubscriptionThread()
def _create_remote_subscriptions(srvObj, stop_evt): """ Creates subscriptions in remote servers so they push their data into ours. """ subscriptions_in_cfg = srvObj.cfg.getSubscriptionsDic() subscriptions = [v for _, v in subscriptions_in_cfg.items()] subscriptions_created = 0 while True: # Done with all of them if not subscriptions: break # Iterate over copy, since we modify the original inside the loop for subscrObj in list(subscriptions): if (not srvObj.getThreadRunPermission()): logger.info("Terminating Subscriber thread ...") return our_host, our_port = srvObj.get_self_endpoint() subs_host, subs_port = subscrObj.getHostId(), subscrObj.getPortNo() # Not subscribing to ourselves if srvObj.is_it_us(subs_host, subs_port): logger.warning( "Skipping subscription to %s:%d because that's us", subs_host, subs_port) return # Create the URL that needs to be set on the remote end so we # get subscribed to it. # TODO: include reverse proxy information when we add support # TODO: hardcoded http will need to be changed when we add support # for https url = 'http://%s:%d/%s' % (our_host, our_port, subscrObj.getUrl() or 'QARCHIVE') logger.info("Creating subscription to %s:%d with url=%s", subs_host, subs_port, url) pars = [["subscr_id", url], ["priority", subscrObj.getPriority()], ["url", url], ["start_date", toiso8601(local=True)]] if subscrObj.getFilterPi(): pars.append(["filter_plug_in", subscrObj.getFilterPi()]) if subscrObj.getFilterPiPars(): pars.append(["plug_in_pars", subscrObj.getFilterPiPars()]) try: # Issue SUBSCRIBE command resp = ngamsHttpUtils.httpGet(subs_host, subs_port, cmd=NGAMS_SUBSCRIBE_CMD, pars=pars) with contextlib.closing(resp): stat = ngamsStatus.to_status(resp, subscrObj.getHostId(), NGAMS_SUBSCRIBE_CMD) if stat.getStatus() != NGAMS_SUCCESS: msg = "Unsuccessful NGAS XML response. Status: %s, message: %s. Will try again later" logger.warning(msg, stat.getStatus(), stat.getMessage()) continue subscriptions_created += 1 logger.info("Successfully subscribed to %s:%d with url=%s", subs_host, subs_port, subscrObj.getUrl()) # Remember to unsubscribe when going Offline. srvObj.getSubscrStatusList().append(subscrObj) subscriptions.remove(subscrObj) except: logger.exception( "Error while adding subscription, will try later") if stop_evt.wait(10): return if subscriptions_created: logger.info("Successfully established %d Subscription(s)", subscriptions_created) else: logger.info("No Subscriptions established")
def _create_remote_subscriptions(srvObj, stop_evt): """ Creates subscriptions in remote servers so they push their data into ours. """ subscriptions_in_cfg = srvObj.cfg.getSubscriptionsDic() subscriptions = [v for _, v in subscriptions_in_cfg.items()] subscriptions_created = 0 def mark_as_active(s): # Removes s from list of subscriptions pending creation, and # ensures the corresponding UNSUBSCRIBE command will be called # at server shutdown srvObj.getSubscrStatusList().append(s) subscriptions.remove(s) while True: # Done with all of them if not subscriptions: break # Iterate over copy, since we modify the original inside the loop for subscrObj in list(subscriptions): if stop_evt.is_set(): logger.info("Terminating Subscriber thread ...") return our_host, our_port = srvObj.get_self_endpoint() subs_host, subs_port = subscrObj.getHostId(), subscrObj.getPortNo() # Not subscribing to ourselves if srvObj.is_it_us(subs_host, subs_port): logger.warning("Skipping subscription to %s:%d because that's us", subs_host, subs_port) continue # Because properly supporting the "Command" configuration mechanism # still requires some more work, we prefer the "SubscriberUrl" # attribute as the main source of URL information. # We still support "Command", but with the following caveats: # * TODO: include reverse proxy information when we add support # * TODO: hardcoded http will need to be changed when we add support # for https # * TODO: fails with IpAddress == '0.0.0.0' url = subscrObj.getUrl() if not url: url = 'http://%s:%d/%s' % (our_host, our_port, subscrObj.command or 'QARCHIVE') logger.info("Creating subscription to %s:%d with url=%s", subs_host, subs_port, url) pars = [["subscr_id", subscrObj.getId()], ["priority", subscrObj.getPriority()], ["url", url], ["start_date", toiso8601(local=True)]] if subscrObj.getFilterPi(): pars.append(["filter_plug_in", subscrObj.getFilterPi()]) if subscrObj.getFilterPiPars(): pars.append(["plug_in_pars", subscrObj.getFilterPiPars()]) try: # Issue SUBSCRIBE command resp = ngamsHttpUtils.httpGet(subs_host, subs_port, cmd=NGAMS_SUBSCRIBE_CMD, pars=pars) with contextlib.closing(resp): stat = ngamsStatus.to_status(resp, subscrObj.getHostId(), NGAMS_SUBSCRIBE_CMD) if stat.getStatus() != NGAMS_SUCCESS: if stat.http_status == 409: short_msg = "Different subscription with ID '%s' already exists, giving up" long_msg = ( "NGAS attempted to create an automatic subscription " "with ID=%s to obtain data from %s:%d, but the remote server " "already has a subscription registered with the same ID, " "but different details.\n\n" "Instead of retrying to create this subscription over and over, " "this server will give up now. To fix this either remove " "the remote subscription, or change the ID of the subscription to be created " "in the local server configuration." ) logger.error(short_msg, subscrObj.getId()) ngamsNotification.notify( srvObj.host_id, srvObj.cfg, NGAMS_NOTIF_ERROR, "Automatic subscription cannot be created", long_msg % (subscrObj.getId(), subs_host, subs_port), force=True) mark_as_active(subscrObj) continue else: msg = "Unsuccessful NGAS XML response. Status: %s, message: %s. Will try again later" logger.warning(msg, stat.getStatus(), stat.getMessage()) continue subscriptions_created += 1 logger.info("Successfully subscribed to %s:%d with url=%s", subs_host, subs_port, subscrObj.getUrl()) mark_as_active(subscrObj) except: logger.exception("Error while adding subscription, will try later") if stop_evt.wait(10): return if subscriptions_created: logger.info("Successfully established %d Subscription(s)", subscriptions_created) else: logger.info("No Subscriptions established")
def process_mirroring_tasks(mirroring_tasks_queue, target_node, ith_thread, n_tasks, srvObj): """ Process mirroring tasks described in the input mirroring_tasks list INPUT: mirroring_tasks_queue Queue of the mirroring tasks assigned to the input server target_node string, Full qualified name of the target node ith_thread int, Thread number n_tasks int, Initial size of the queue srvObj ngamsServer, Reference to NG/AMS server class object RETURNS: Void """ logger.debug("Inside mirror worker worker %d to mirror files to %s", ith_thread, target_node) try: # Loop on the mirroring_tasks_queue while 1: # Get tasks from queue if (len(mirroring_tasks_queue)): try: if (ith_thread % 2): item = mirroring_tasks_queue.pop() else: item = mirroring_tasks_queue.popleft() except: break else: break # get all the fields we're interested in and form them into a command to fetch the files. # these fields were extracted in get_list_mirroring_tasks() logger.debug("next task: %r", item) staging_file = str(item[1]) mimeType = str(item[3]) checksum = str(item[4]) rowid = str(item[2]) file_id = str(item[9]) fileSize = str(item[0]) fileInfo = {} fileInfo['sourceHost'] = str(item[5]) fileInfo['diskId'] = str(item[6]) fileInfo['hostId'] = str(item[7]) fileInfo['fileVersion'] = str(item[8]) fileInfo['fileId'] = file_id logger.info( "Processing mirroring task (Target node: %s, file size: %s, Thread: %d) file info: %s", target_node, fileSize, ith_thread, str(fileInfo)) # Initialize ngamsReqProps object by just specifing the fileURI and the mime type reqPropObj = ngamsReqProps.ngamsReqProps() reqPropObj.setMimeType(mimeType) reqPropObj.checksum = checksum reqPropObj.fileinfo = fileInfo reqPropObj.setSize(fileSize) # Start clock start = time.time() (stgFilename, targetDiskInfo) = calculateStagingName(srvObj, file_id, staging_file) reqPropObj.setStagingFilename(stgFilename) reqPropObj.setTargDiskInfo(targetDiskInfo) try: # Construct query to update ingestion date, ingestion time and status query = "update ngas_mirroring_bookkeeping set status='FETCHING', " query += "staging_file = {0}, " query += "attempt = nvl(attempt + 1, 1) " query += "where rowid = {1}" # Add query to the queue srvObj.getDb().query2(query, args=(reqPropObj.getStagingFilename(), rowid)) logger.info("Mirroring file: %s", file_id) ngamsCmd_MIRRARCHIVE.handleCmd(srvObj, reqPropObj) status = "SUCCESS" except ngamsFailedDownloadException.FailedDownloadException: # Something bad happened... logger.exception("Failed to fetch %s", file_id) status = "FAILURE" except ngamsFailedDownloadException.AbortedException: logger.warning("File fetch aborted: %s", file_id) status = "ABORTED" except ngamsFailedDownloadException.PostponeException: logger.exception( "Failed to fetch %s - will try to resume on next iteration", file_id) status = "TORESUME" except Exception: # this clause should never be reached logger.exception("Fetch failed in an unexpected way") status = "FAILURE" # Get time elapsed elapsed_time = (time.time() - start) # Construct query to update ingestion date, ingestion time and status query = "update ngas_mirroring_bookkeeping set status = {0}," if (status != 'TORESUME'): query += "staging_file = null, " query += "ingestion_date = {1}," query += "ingestion_time = nvl(ingestion_time, 0.0) + {2} " query += "where rowid = {3}" args = (status, toiso8601(fmt=FMT_DATETIME_NOMSEC) + ":000", elapsed_time, rowid) srvObj.getDb().query2(query, args=args) # Log message for mirroring task processed completion = 100 * (n_tasks - len(mirroring_tasks_queue)) / float(n_tasks) logger.info( "Mirroring task (Target node: %s Thread: %d) processed in %fs (%s), completion: %f%%: %s", target_node, ith_thread, elapsed_time, status, completion, str(fileInfo)) logger.info("Mirroring Worker complete") except Exception: logger.exception("Error while running mirroring worker") # Return Void return
def ngamsGeneric(srvObj, reqPropsObj): """ Data Archiving Plug-In to handle archiving of SDM multipart related message files containing ALMA UIDs in the Content-Location mime parameter or any other kind of file srvObj: Reference to NG/AMS Server Object (ngamsServer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: Standard NG/AMS Data Archiving Plug-In Status as generated by: ngamsPlugInApi.genDapiSuccessStat() (ngamsDapiStatus). """ logger.debug("Mirroring plug-in handling data for file: %s", os.path.basename(reqPropsObj.getFileUri())) # Create the file diskInfo = reqPropsObj.getTargDiskInfo() stagingFilename = reqPropsObj.getStagingFilename() ext = os.path.splitext(stagingFilename)[1][1:] # reqPropsObj format: /MIRRARCHIVE?mime_type=application/x-tar&filename=... if (reqPropsObj.getMimeType()): format = reqPropsObj.getMimeType() else: errMsg = "mime_type not specified in MIRRARCHIVE request" raise Exception, errMsg # File Uri format: http://ngasbe03.aiv.alma.cl:7777/RETRIEVE?disk_id=59622720f79296473f6106c15e5c2240&host_id=ngasbe03:7777&quick_location=1&file_version=1&file_id=backup.2011-02-02T22:01:59.tar # Get file id fileVersion = reqPropsObj.fileinfo['fileVersion'] fileId = reqPropsObj.fileinfo['fileId'] # Specific treatment depending on the mime type if ((format.find("multipart") >= 0) or (format.find("multialma") >= 0)): logger.debug("applying plug-in specific treatment") fo = open(stagingFilename, "r") try: (fileId, finalName, format) = specificTreatment(fo) finally: fo.close() else: finalName = fileId logger.debug( "File with URI %s is being handled by ngamsGeneric: format=%s file_id=%s file_version=%s finalName=%s", reqPropsObj.getFileUri(), format, fileId, fileVersion, finalName) try: # Compression parameters uncomprSize = ngamsPlugInApi.getFileSize(stagingFilename) compression = "" # File name and paths date = toiso8601(fmt=FMT_DATE_ONLY) relPath, relFilename, complFilename, fileExists = genFileInfo( srvObj.getDb(), srvObj.getCfg(), reqPropsObj, diskInfo, stagingFilename, fileId, fileVersion, finalName, [date]) # Make sure the format is defined if not format: format = ngamsPlugInApi.determineMimeType(srvObj.getCfg(), stagingFilename) # FileSize fileSize = ngamsPlugInApi.getFileSize(stagingFilename) # Return resDapi object return ngamsPlugInApi.genDapiSuccessStat( diskInfo.getDiskId(), relFilename, fileId, fileVersion, format, fileSize, uncomprSize, compression, relPath, diskInfo.getSlotId(), fileExists, complFilename) except Exception, e: err = "Problem processing file in stagging area: " + str(e) errMsg = genLog("NGAMS_ER_DAPI_BAD_FILE", [stagingFilename, _PLUGIN_ID, err]) raise Exception, errMsg
def _remStatErrReport(srvObj, reqPropsObj, tmpFilePat, filesMisCopiesDbmName, filesNotRegDbmName, fileListDbmName, diskId=None, fileId=None, fileVersion=None): """ Function to generate an error report in case of the REMDISK and REMFILE commands. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). tmpFilePat: File pattern to be used when generating temporary files (string). filesMisCopiesDbmName: Name of DBM containing references to files with less than the required number of copies (string). filesNotRegDbmName: Name of DBM containing complete filenames as keys referring to files, which are not registered in the DB (string). fileListDbmName: Name of DBM containing all information about the files concerned with the query. diskId: Disk ID if relevant (string). fileId: File ID if relevant (string). fileVersion: File Version if relevant (integer). Returns: In case inconsistencies were found, an ngamsStatus object with more information is returned. Otherwise, if everything was found to be OK, None is returned (ngamsStatus|None). """ cmd = reqPropsObj.getCmd() hostId = srvObj.getHostId() ######################################################################### # At least three copies of each file? ######################################################################### filesMisCopyDbm = ngamsDbm.ngamsDbm(filesMisCopiesDbmName, writePerm=0) misFileCopies = filesMisCopyDbm.getCount() if (misFileCopies and reqPropsObj.hasHttpPar("notif_email")): xmlReport = 0 if (xmlReport): # TODO: pass else: statRep = tmpFilePat + "_NOTIF_EMAIL.txt" fo = open(statRep, "w") tmpFormat = cmd + " STATUS REPORT - " +\ "MISSING FILE COPIES:\n\n" +\ "==Summary:\n\n" +\ "Date: %s\n" +\ "NGAS Host ID: %s\n" +\ "Disk ID: %s\n" +\ "Files Detected: %d\n\n" +\ "==File List:\n\n" fo.write(tmpFormat % (toiso8601(), hostId, diskId, misFileCopies)) tmpFormat = "%-32s %-12s %-6s\n" fo.write(tmpFormat % ("File ID", "Version", "Copies")) fo.write(tmpFormat % (32 * "-", 7 * "-", 6 * "-")) # Loop over the files an generate the report. filesMisCopyDbm.initKeyPtr() while (1): fileKey, fileInfo = filesMisCopyDbm.getNext() if (not fileKey): break noOfCopies = fileInfo.getTag().split(": ")[-1] fo.write(tmpFormat % (fileInfo.getFileId(), str( fileInfo.getFileVersion()), noOfCopies)) fo.write(52 * "-") fo.write("\n\n==END\n") fo.close() mimeType = NGAMS_TEXT_MT # Send out Notification Email. _notify(srvObj, reqPropsObj, statRep, mimeType, cmd) del filesMisCopyDbm # Generate error message if files with less than the required number # of copies are concerned by the query. if (misFileCopies): errMsg = genLog("NGAMS_WA_FILE_COPIES") status = srvObj.genStatus(NGAMS_FAILURE, errMsg) status.setMessage(status.getMessage() +\ " Cannot remove item: %s/%s/%s" % (str(diskId), str(fileId), str(fileVersion))) logger.warning(errMsg) return status ######################################################################### ######################################################################### # Check for spurious files. ######################################################################### spuriousFilesDbmName = checkSpuriousFiles(srvObj, tmpFilePat, hostId, diskId, fileId, fileVersion) spuriousFilesDbm = ngamsDbm.ngamsDbm(spuriousFilesDbmName, writePerm=0) spuriousFiles = spuriousFilesDbm.getCount() srvDataChecking = srvObj.getDb().getSrvDataChecking(hostId) if (spuriousFiles and reqPropsObj.hasHttpPar("notif_email")): xmlReport = 0 if (xmlReport): # TODO: pass else: statRep = tmpFilePat + "_NOTIF_EMAIL.txt" fo = open(statRep, "w") tmpFormat = cmd + " STATUS REPORT - SPURIOUS FILES:\n\n" +\ "==Summary:\n\n" +\ "Date: %s\n" +\ "NGAS Host ID: %s\n" +\ "Disk ID: %s\n" +\ "Spurious Files: %d\n" fo.write(tmpFormat % (toiso8601(), hostId, diskId, spuriousFiles)) if (srvDataChecking): fo.write("Note: NGAS Host is performing Data Consistency " +\ "Checking - consider to switch off!\n") fo.write("\n==File List:\n\n") tmpFormat = "%-32s %-12s %-11s %-11s\n" fo.write(tmpFormat % ("File ID", "File Version", "File Status", "Ignore Flag")) fo.write(tmpFormat % (32 * "-", 12 * "-", 11 * "-", 11 * "-")) # Loop over the files an generate the report. spuriousFilesDbm.initKeyPtr() while (1): key, fileInfo = spuriousFilesDbm.getNext() if (not key): break fileId = fileInfo[ngamsDbCore.SUM1_FILE_ID] fileVersion = str(fileInfo[ngamsDbCore.SUM1_VERSION]) fileStatus = fileInfo[ngamsDbCore.SUM1_FILE_STATUS] ignoreFlag = str(fileInfo[ngamsDbCore.SUM1_FILE_IGNORE]) fo.write(tmpFormat % (fileId, fileVersion, fileStatus, ignoreFlag)) fo.write("\n\n==END\n") fo.close() mimeType = NGAMS_TEXT_MT # Send out Notification Email. _notify(srvObj, reqPropsObj, statRep, mimeType, cmd) del spuriousFilesDbm # Generate error message if spurious files were found. if (spuriousFiles): errMsg = genLog("NGAMS_WA_FILE_NON_REM") logger.warning(errMsg) status = srvObj.genStatus(NGAMS_FAILURE, errMsg) msg = status.getMessage() + " Cannot delete specified files" if (srvDataChecking): msg += ". Note: NGAS Host is performing Data Consistency " +\ "Checking - consider to switch off!" status.setMessage(msg) return status ######################################################################### ######################################################################### # Unregistered files found? ######################################################################### filesNotRegDbm = ngamsDbm.ngamsDbm(filesNotRegDbmName, writePerm=0) unRegFilesFound = filesNotRegDbm.getCount() if (unRegFilesFound and reqPropsObj.hasHttpPar("notif_email")): xmlReport = 0 if (xmlReport): # TODO: pass else: statRep = tmpFilePat + "_NOTIF_EMAIL.txt" fo = open(statRep, "w") tmpFormat = cmd + " STATUS REPORT - " +\ "NON-REGISTERED FILES:\n\n" +\ "==Summary:\n\n" +\ "Date: %s\n" +\ "NGAS Host ID: %s\n" +\ "Disk ID: %s\n" +\ "Non-Registered Files: %d\n\n" +\ "==File List:\n\n" fo.write(tmpFormat % (toiso8601(), hostId, diskId, unRegFilesFound)) # Loop over the files an generate the report. filesNotRegDbm.initKeyPtr() while (1): fileName, dummy = filesNotRegDbm.getNext() if (not fileName): break fo.write(fileName + "\n") fo.write("\n\n==END\n") fo.close() mimeType = NGAMS_TEXT_MT # Send out Notification Email. _notify(srvObj, reqPropsObj, statRep, mimeType, cmd) del filesNotRegDbm # Generate error message if non-registered files found. if (unRegFilesFound): errMsg = genLog("NGAMS_WA_FILES_NOT_REG") logger.warning(errMsg) status = srvObj.genStatus(NGAMS_FAILURE, errMsg) status.setMessage(status.getMessage() +\ " Cannot remove volume: %s/%s/%s" % (str(diskId), str(fileId), str(fileVersion))) return status ######################################################################### ######################################################################### # Check if the files to be removed actually can be removed. ######################################################################### nonRemFilesDbmName = checkFilesRemovable(srvObj, fileListDbmName, tmpFilePat) nonRemFilesDbm = ngamsDbm.ngamsDbm(nonRemFilesDbmName, writePerm=0) nonRemFiles = nonRemFilesDbm.getCount() if (nonRemFiles and reqPropsObj.hasHttpPar("notif_email")): xmlReport = 0 if (xmlReport): # TODO: pass else: statRep = tmpFilePat + "_NOTIF_EMAIL.txt" fo = open(statRep, "w") tmpFormat = cmd + " STATUS REPORT - " +\ "NON-REMOVABLE FILES:\n\n" +\ "==Summary:\n\n" +\ "Date: %s\n" +\ "NGAS Host ID: %s\n" +\ "Disk ID: %s\n" +\ "Non-Removable Files: %d\n\n" +\ "==File List:\n\n" fo.write(tmpFormat % (toiso8601(), hostId, diskId, nonRemFiles)) tmpFormat = "%-32s %-12s %-32s\n" fo.write(tmpFormat % ("File ID", "File Version", "Filename")) fo.write(tmpFormat % (32 * "-", 12 * "-", 32 * "-")) # Loop over the files an generate the report. nonRemFilesDbm.initKeyPtr() while (1): key, fileInfo = nonRemFilesDbm.getNext() if (not key): break fo.write(tmpFormat % (fileInfo[ngamsDbCore.SUM1_FILE_ID], fileInfo[ngamsDbCore.SUM1_VERSION], fileInfo[ngamsDbCore.SUM1_FILENAME])) fo.write("\n\n==END\n") fo.close() mimeType = NGAMS_TEXT_MT # Send out Notification Email. _notify(srvObj, reqPropsObj, statRep, mimeType, cmd) del nonRemFilesDbm # Generate error message if files concerned by the query could not be # deleted. if (nonRemFiles): errMsg = genLog("NGAMS_WA_FILE_NON_REM") logger.warning(errMsg) status = srvObj.genStatus(NGAMS_FAILURE, errMsg) status.setMessage(status.getMessage() +\ " Cannot delete specified files") return status ######################################################################### # If we got to this point, no problems where found. return None
def _registerExec(srvObj, fileListDbmName, tmpFilePat, diskInfoDic, reqPropsObj=None): """ Register the files listed in the File List DBM (ngamsDbm), which match the mime-type(s) either specified in the 'mimeType' parameter, or if this is not specified, which match all the mime-types specified in the configuration file. When the registration procedure has been executed, the function sends an Email Notification message indicating which files were registered if the HTTP parameter 'notif_email' is given. The functions creates a File Info Objects per file handled and writes this in a temporary file, which is a DBM file. The keys in this DB is simply the file number in the sequence of files handled, pointing to a pickled ngamsFileInfo object. Each of the File Info Objects indicates if the file was registered or not. This is done by setting the tag of the File Info Object to one of the following values: REGISTERED: The file was successfully registered in the NGAS DB FAILED: The file was selected for registration but could not be properly registered because of inconsistencies. The status will be of the format: 'FAILED[: <reason>]'. REJECTED: A file found under the specified path directory was not accepted for cloning, usually because the mime-type was not correct. The status will be of the format: 'REJECTED[: <reason>]'. Note, that registration is comparable to archiving of files. For that reason a DAPI must be provided for each type of file that should be registered. If this is not fullfilled, the file registration will fail. Only files stored on one of the NGAS disks configured in the configuration file, are considered. Files stored in other locations are ignored. srvObj: Instance of NG/AMS Server object (ngamsServer). fileListDbmName: Name of a DBM containing the information about the files to be registered. Each element in the list is referred to by a key, which is a number. These points to a pickled list for each file containing the following information: [<Filename>, <Disk ID>, <Mime-Type>] The information for each disk concerned is also contained in the DB referred to by its Disk ID and by its mount point. The data is a pickled instance of the ngamsDiskInfo class (string). tmpFilePat: Pattern for temporary files used during the registration process (string). diskInfoDic: Dictionary with Disk IDs as keys pointing to the info about the disk (dictionary/ngamsDiskInfo). reqPropsObj: If an NG/AMS Request Properties Object is given, the Request Status will be updated as the request is carried out (ngamsReqProps). Returns: Void. """ emailNotif = 0 if (reqPropsObj): if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1 # Create the temporary BSD DB to contain the information for the # Email Notification Message. if (emailNotif): regDbmName = tmpFilePat + "_NOTIF_EMAIL" regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm=1) # Open the DBM containing the list of files to (possibly) register. fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1) # Want to parse files in alphabetical order. # TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting. tmpFileList = tmpFilePat + "_FILE_LIST" rmFile(tmpFileList) with open(tmpFileList, "wb") as fo: fileListDbm.initKeyPtr() while (1): dbmKey, fileInfo = fileListDbm.getNext() if (not dbmKey): break fo.write(dbmKey + b"\n") sortFileList = tmpFilePat + "_SORT_FILE_LIST" rmFile(sortFileList) shellCmd = "sort %s > %s" % (tmpFileList, sortFileList) stat, out, err = ngamsCore.execCmd(shellCmd) if (stat != 0): raise Exception("Error executing command: %s. Error: %s, %s" %\ (shellCmd, str(out), str(err))) rmFile(tmpFileList) # Go through each file in the list, check if the mime-type is among the # ones, which apply for registration. If yes try to register the file # by invoking the corresponding DAPI on the file. fileRegCount = 0 fileFailCount = 0 fileRejectCount = 0 regTimeAccu = 0.0 fileCount = 0 fo = open(sortFileList) run = 1 while (run): reg_start = time.time() dbmKey = fo.readline() if (dbmKey.strip() == ""): run = 0 continue fileInfo = fileListDbm.get(dbmKey[0:-1]) filename = fileInfo[0] diskId = fileInfo[1] mimeType = fileInfo[2] # Register the file. Check first, that exactly this file is # not already registered. In case it is, the file will be rejected. regPi = srvObj.getCfg().register_plugins[mimeType] logger.debug("Plugin found for %s: %s", mimeType, regPi) params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars) tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\ setMimeType(mimeType).\ setStagingFilename(filename).\ setTargDiskInfo(diskInfoDic[diskId]).\ setHttpMethod(NGAMS_HTTP_GET).\ setCmd(NGAMS_REGISTER_CMD).\ setSize(os.path.getsize(filename)).\ setFileUri(filename).\ setNoReplication(1) tmpFileObj = ngamsFileInfo.ngamsFileInfo() try: # Invoke Registration Plug-In. piName = regPi.name plugInMethod = loadPlugInEntryPoint(piName) piRes = plugInMethod(srvObj, tmpReqPropsObj, params) del tmpReqPropsObj # Check if this file is already registered on this disk. In case # yes, it is not registered again. files = srvObj.db.getFileSummary1(srvObj.getHostId(), [piRes.getDiskId()], [piRes.getFileId()]) fileRegistered = 0 for tmpFileInfo in files: tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT] tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME] tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\ tmpFilename) if (tmpComplFilename == filename): fileRegistered = 1 break if (fileRegistered): fileRejectCount += 1 tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\ "and path: %s is already registered on disk " +\ "with Disk ID: %s" tmpMsg = tmpMsgForm % (piRes.getFileId(), piRes.getFileVersion(), filename, piRes.getDiskId()) logger.warning(tmpMsg + ". File is not registered again.") if (emailNotif): tmpFileObj.\ setDiskId(diskId).setFilename(filename).\ setTag(tmpMsg) regDbm.addIncKey(tmpFileObj) if (reqPropsObj): reqPropsObj.incActualCount(1) ngamsHighLevelLib.stdReqTimeStatUpdate( srvObj, reqPropsObj, regTimeAccu) regTimeAccu += time.time() - reg_start fileCount += 1 continue # Calculate checksum. We maintain the old name for backwards # compatibility crc_variant = srvObj.cfg.getCRCVariant() if crc_variant == ngamsFileUtils.CHECKSUM_CRC32_INCONSISTENT: crc_variant = 'ngamsGenCrc32' checksum = ngamsFileUtils.get_checksum(65536, filename, crc_variant) or '' # Move file and update information about file in the NGAS DB. mvFile(filename, piRes.getCompleteFilename()) ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum, crc_variant) ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes) ngamsLib.makeFileReadOnly(piRes.getCompleteFilename()) if (emailNotif): uncomprSize = piRes.getUncomprSize() ingestDate = time.time() creDateSecs = getFileCreationTime(filename) tmpFileObj.\ setDiskId(diskId).\ setFilename(filename).\ setFileId(piRes.getFileId()).\ setFileVersion(piRes.getFileVersion()).\ setFormat(piRes.getFormat()).\ setFileSize(piRes.getFileSize()).\ setUncompressedFileSize(uncomprSize).\ setCompression(piRes.getCompression()).\ setIngestionDate(ingestDate).\ setIgnore(0).\ setChecksum(checksum).\ setChecksumPlugIn(crc_variant).\ setFileStatus(NGAMS_FILE_STATUS_OK).\ setCreationDate(creDateSecs).\ setTag("REGISTERED") fileRegCount += 1 # If running as a cache archive, update the Cache New Files DBM # with the information about the new file. if (srvObj.getCachingActive()): fileVer = fio.getFileVersion() ngamsCacheControlThread.addEntryNewFilesDbm( srvObj, diskId, piRes.getFileId(), fileVer, filename) # Generate a confirmation log entry. msg = genLog("NGAMS_INFO_FILE_REGISTERED", [ filename, piRes.getFileId(), piRes.getFileVersion(), piRes.getFormat() ]) time.sleep(0.005) regTime = time.time() - reg_start msg = msg + ". Time: %.3fs." % (regTime) logger.info(msg, extra={'to_syslog': 1}) except Exception as e: errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)]) logger.error(errMsg) if (emailNotif): tmpFileObj.\ setDiskId(diskId).setFilename(filename).\ setTag(errMsg) fileFailCount += 1 regTime = time.time() - reg_start # TODO (rtobar, 2016-01): Why don't we raise an exception here? # Otherwise the command appears as successful on the # client-side # Add the file information in the registration report. if (emailNotif): regDbm.addIncKey(tmpFileObj) # Update request status time information. regTimeAccu += regTime if (reqPropsObj): reqPropsObj.incActualCount(1) ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj, regTimeAccu) fileCount += 1 fo.close() rmFile(sortFileList) if (emailNotif): regDbm.sync() del fileListDbm rmFile(fileListDbmName + "*") # Final update of the Request Status. if (reqPropsObj): if (reqPropsObj.getExpectedCount() and reqPropsObj.getActualCount()): complPercent = (100.0 * (float(reqPropsObj.getActualCount()) / float(reqPropsObj.getExpectedCount()))) else: complPercent = 100.0 reqPropsObj.setCompletionPercent(complPercent, 1) reqPropsObj.setCompletionTime(1) srvObj.updateRequestDb(reqPropsObj) # Send Register Report with list of files cloned to a possible # requestor(select) of this. if (emailNotif): xmlStat = 0 if (xmlStat): # Create an instance of the File List Class, used to store the # Registration Report. regStat = ngamsFileList.\ ngamsFileList("FILE_REGISTRATION_STATUS", "Status report for file " +\ "registration") # Loop over the file objects in the BSD DB and add these # in the status object. regDbm.initKeyPtr() while (1): key, tmpFileObj = regDbm.getNext() if (not key): break regStat.addFileInfoObj(tmpFileObj) # Set overall status of registration procedure. regStat.setStatus("Files Found: " + str(fileCount + 1) + ", "+\ "Files Registered: " + str(fileRegCount) +\ ", " +\ "Files Failed: " + str(fileFailCount) + ", " +\ "Files Rejected: " + str(fileRejectCount)) status = srvObj.genStatus(NGAMS_SUCCESS, "REGISTER command status report").\ addFileList(regStat) statRep = status.genXmlDoc() statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep) mimeType = NGAMS_XML_MT else: # Generate a 'simple' ASCII report. statRep = tmpFilePat + "_NOTIF_EMAIL.txt" fo = open(statRep, "w") if (reqPropsObj): path = reqPropsObj.getHttpPar("path") else: path = "-----" if (fileCount): timePerFile = (regTimeAccu / fileCount) else: timePerFile = 0 tmpFormat = "REGISTER STATUS REPORT:\n\n" +\ "==Summary:\n\n" +\ "Date: %s\n" +\ "NGAS Host: %s\n" +\ "Search Path: %s\n" +\ "Total Number of Files: %d\n" +\ "Number of Registered Files: %d\n" +\ "Number of Failed Files: %d\n" +\ "Number of Rejected Files: %d\n" +\ "Total processing time (s): %.3f\n" +\ "Handling time per file (s): %.3f\n\n" +\ "==File List:\n\n" fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), path, fileCount, fileRegCount, fileFailCount, fileRejectCount, regTimeAccu, timePerFile)) tmpFormat = "%-80s %-32s %-3s %-10s\n" fo.write(tmpFormat %\ ("Filename", "File ID", "Ver", "Status")) fo.write(tmpFormat % (80 * "-", 32 * "-", 3 * "-", 10 * "-")) regDbm.initKeyPtr() while (1): key, tmpFileObj = regDbm.getNext() if (not key): break mtPt = diskInfoDic[tmpFileObj.getDiskId()].getMountPoint() filename = os.path.normpath(mtPt + "/" +\ tmpFileObj.getFilename()) line = tmpFormat %\ (filename, tmpFileObj.getFileId(), str(tmpFileObj.getFileVersion()), tmpFileObj.getTag()) fo.write(line) fo.write(128 * "-") fo.write("\n\n==END\n") fo.close() mimeType = NGAMS_TEXT_MT # Send out the status report. emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",") attachmentName = "RegisterStatusReport" if (reqPropsObj.hasHttpPar("path")): attachmentName += "-" + reqPropsObj.getHttpPar("path").\ replace("/", "_") ngamsNotification.notify(srvObj.host_id, srvObj.cfg, NGAMS_NOTIF_INFO, "REGISTER STATUS REPORT", statRep, recList=emailAdrList, force=1, contentType=mimeType, attachmentName=attachmentName) del regDbm rmFile(regDbmName + "*") rmFile(statRep) # Generate final status log + exit. if (fileCount > 0): timePerFile = (regTimeAccu / fileCount) else: timePerFile = 0.0 msg = "Registration procedure finished processing Register Request - " + \ "terminating. Files handled: %d. Total time: %.3fs. " + \ "Average time per file: %.3fs." logger.debug(msg, fileCount, regTimeAccu, timePerFile)