def test_VolumeDir_01(self):
"""
Synopsis:
Grouping of data volumes under the Volume Dir in the NGAS Root Dir.
Description:
See ngamsArchiveCmdTest.test_VolumeDir_01().
This tests verifies that the files archived into this structure can
be received.
Expected Result:
When the server goes Online, it should accept the given directory
structure and it should be possible to archive files into this
structureand subsequently to receive them.
Test Steps:
- Create the volume dirs from an existing structure.
- Start server with configuration specifying the Volumes Dir in which
all volumes will be hosted.
- Archive a FITS file.
- Retrieve the files and check that they are OK.
Remarks:
...
"""
# Create basic structure.
ngasRootDir = tmp_path("NGAS")
rmFile(ngasRootDir)
checkCreatePath(ngasRootDir)
subprocess.check_call(
['tar', 'zxf',
self.resource('src/volumes_dir.tar.gz')])
mvFile('volumes', ngasRootDir)
# Create configuration, start server.
self.prepExtSrv(delDirs=0, cfgFile='src/ngamsCfg_VolumeDirectory.xml')
# Archive a file.
stat = self.archive("src/SmallFile.fits")
self.assert_status_ref_file(
"ref/ngamsRetrieveCmdTest_test_VolumeDir_01_01_ref", stat)
# Check that the target files have been archived in their
# appropriate locations.
trgFile = tmp_path("test_VolumeDir_01_tmp")
refFile = "src/SmallFile.fits"
outFilePath = tmp_path("SmallFile.fits")
stat = self.retrieve("TEST.2001-05-08T15:25:00.123",
targetFile=trgFile)
# unzip the the file and diff against original
unzip(trgFile, outFilePath)
self.checkFilesEq(refFile, outFilePath, "Retrieved file incorrect")
def processRequest(srvObj,
reqPropsObj,
httpRef,
fileInfoObj,
trgDiskInfoObj):
"""
Process the Rearchive Request.
For a description of the parameters: Check handleCmd().
fileInfoObj: File information for file to be restored (ngamsFileInfo).
trgDiskInfoObj: Disk Info Object for target disk (ngamsDiskInfo).
Returns: Void.
"""
# Generate the DB File Information.
newFileInfoObj = fileInfoObj.clone().\
setDiskId(trgDiskInfoObj.getDiskId()).\
setCreationDate(time.time())
# Generate the final storage location and move the file there.
targetFilename = os.path.normpath("%s/%s" %\
(trgDiskInfoObj.getMountPoint(),
newFileInfoObj.getFilename()))
logger.debug("Move Restore Staging File to final destination: %s->%s ...",
reqPropsObj.getStagingFilename(), targetFilename)
io_start = time.time()
mvFile(reqPropsObj.getStagingFilename(), targetFilename)
ioTime = time.time() - io_start
reqPropsObj.incIoTime(ioTime)
logger.debug("Moved Restore Staging File to final destination: %s->%s",
reqPropsObj.getStagingFilename(), targetFilename)
# Update the DB with the information about the new file.
# Update information for Main File/Disk in DB.
newFileInfoObj.write(srvObj.getHostId(), srvObj.getDb())
diskSpace = getDiskSpaceAvail(trgDiskInfoObj.getMountPoint())
newSize = (trgDiskInfoObj.getBytesStored() + newFileInfoObj.getFileSize())
ioTime = (trgDiskInfoObj.getTotalDiskWriteTime() + reqPropsObj.getIoTime())
trgDiskInfoObj.\
setNumberOfFiles(trgDiskInfoObj.getNumberOfFiles() + 1).\
setAvailableMb(diskSpace).setBytesStored(newSize).\
setTotalDiskWriteTime(ioTime).write(srvObj.getDb())
# Ensure that the restored file is readonly.
ngamsLib.makeFileReadOnly(targetFilename)
# Add the file info object for the target file in the disk info object.
trgDiskInfoObj.addFileObj(newFileInfoObj)
def run(srvObj, stopEvt):
cfg = srvObj.getCfg()
logdir = os.path.dirname(cfg.getLocalLogFile())
# The LOG-ROTATE-<timestamp>.nglog.unsaved pattern is produced by the
# ngamsServer.NgasRotatingFileHandler class, which is the file handler
# attached to the root logger in the server process
logger.debug("Checking if there are unsaved rotated logfiles")
for unsaved in sorted(
glob.glob(os.path.join(logdir, 'LOG-ROTATE-*.nglog.unsaved'))):
# Remove the .unsaved bit, leave the rest
fname = '.'.join(unsaved.split('.')[:-1])
try:
mvFile(unsaved, fname)
try_archiving(cfg, srvObj, fname)
except Exception as e:
mvFile(fname, unsaved)
if isinstance(e, socket.error) and e.errno == errno.ECONNREFUSED:
# this is expected when the server is just starting up,
# let's ignore for now
logger.warning(
"Server not up yet, postponing processing of %s", unsaved)
continue
raise
# Do additional things with our logfiles
for plugin in get_logfile_handler_plugins(cfg):
try:
plugin(srvObj, fname)
except:
logger.exception("Error while handling logfile %s", fname)
logger.debug("Check if there are old rotated logfiles to remove ...")
max_rotations = max(min(cfg.getLogRotateCache(), 100), 0)
logfiles = glob.glob(os.path.join(logdir, 'LOG-ROTATE-*.nglog'))
logfiles.sort()
for f in logfiles[max_rotations:]:
logger.info("Removing rotated logfile %s", f)
rmFile(f)
def ngamsTileCompress(filename):
"""
Tile compress the referenced file.
filename: Filename (string).
Returns: Void.
"""
tmpFilename = filename + ".tmp"
try:
comprCmd = "imcopy %s '%s[compress]'" % (filename, tmpFilename)
logger.debug("Command to tile compress file: %s", comprCmd)
stat, out = commands.getstatusoutput(comprCmd)
if (stat != 0):
msg = "Error compressing file: %s. Error: %s" %\
(filename, stat.replace("\n", " "))
raise Exception, msg
mvFile(tmpFilename, filename)
logger.debug("Successfully tile compressed file: %s", filename)
except Exception, e:
rmFile(tmpFilename)
raise Exception, e
def run(srvObj, stopEvt):
cfg = srvObj.getCfg()
logdir = os.path.dirname(cfg.getLocalLogFile())
# The LOG-ROTATE-<timestamp>.nglog.unsaved pattern is produced by the
# ngamsServer.NgasRotatingFileHandler class, which is the file handler
# attached to the root logger in the server process
logger.debug("Checking if there are unsaved rotated logfiles")
for unsaved in glob.glob(os.path.join(logdir,
'LOG-ROTATE-*.nglog.unsaved')):
# Remove the .unsaved bit, leave the rest
fname = '.'.join(unsaved.split('.')[:-1])
mvFile(unsaved, fname)
# Connect to the server and send a pull ARCHIVE request
if cfg.getArchiveRotatedLogfiles():
file_uri = "file://" + fname
host, port = srvObj.get_self_endpoint()
ngamsPClient.ngamsPClient(host,
port).archive(file_uri, 'ngas/nglog')
# Do additional things with our logfiles
for plugin in get_logfile_handler_plugins(cfg):
try:
plugin(srvObj, fname)
except:
logger.exception("Error while handling logfile %s", fname)
logger.debug("Check if there are old rotated logfiles to remove ...")
max_rotations = max(min(cfg.getLogRotateCache(), 100), 0)
logfiles = glob.glob(os.path.join(logdir, 'LOG-ROTATE-*.nglog'))
logfiles.sort()
for f in logfiles[max_rotations:]:
logger.info("Removing rotated logfile %s", f)
rmFile(f)
logger.warning(
"the fetch has already downloaded data. marking as TORESUME")
raise ngamsFailedDownloadException.PostponeException(e)
else:
logger.warning(
3, "no data has been downloaded yet. Marking as FAILURE")
raise ngamsFailedDownloadException.FailedDownloadException(e)
# Invoke DAPI
logger.info("Invoking DAPI")
resDapi = ngamsDAPIMirroring.ngamsGeneric(srvObj, reqPropsObj)
# Move file to final destination.
logger.info("Moving file to final destination: %s",
resDapi.getCompleteFilename())
ioTime = mvFile(reqPropsObj.getStagingFilename(),
resDapi.getCompleteFilename())
reqPropsObj.incIoTime(ioTime)
# Check/generate remaining file info + update in DB.
logger.info("Creating db entry")
creDate = srvObj.getDb().convertTimeStamp(
getFileCreationTime(resDapi.getCompleteFilename()))
sqlUpdate = "update ngas_disks set available_mb = available_mb - {0} / (1024 * 1024), bytes_stored = bytes_stored + {1} " +\
"where disk_id = {2}"
srvObj.getDb().query2(sqlUpdate,
args=(resDapi.getFileSize(), resDapi.getFileSize(),
resDapi.getDiskId()))
ts = srvObj.getDb().convertTimeStamp(time.time())
sqlQuery = "INSERT INTO ngas_files " +\
"(disk_id, file_name, file_id, file_version, " +\
def _cloneExplicit(srvObj, reqPropsObj, diskId, fileId, fileVersion,
targetDiskId):
"""
Execute CLONE Command, where the source Disk ID, File ID and File Version
are specified. Is much faster than a normal CLONE Command when an explicit
file is specified.
srvObj: Reference to instance of Server Object (ngamsServer).
fileInfoObj: File info object with info about the file
diskId: ID of disk hosting the file to be cloned (string).
fileId: ID of file to clone (string).
fileVersion: Version of file to clone (integer).
targetDiskId: ID of target disk (string).
Returns: Void.
"""
T = TRACE(1)
# Resolve the location of the file to clone.
location, hostId, ipAddress, portNo, mountPoint, filename,\
fileVersion, mimeType =\
ngamsFileUtils.quickFileLocate(srvObj, reqPropsObj, fileId,
diskId=diskId,
fileVersion=fileVersion)
# Read also the entire file info (unfortunately).
srcFileInfo = ngamsFileInfo.ngamsFileInfo().read(srvObj.getHostId(),
srvObj.getDb(), fileId,
fileVersion, diskId)
# Determine target disk.
if (targetDiskId == ""):
# Try to find a disk not hosting already a file with that
# ID + version.
diskExemptList = [diskId]
while (1):
trgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
mimeType, 1, diskExemptList)
# Check if a file with that ID + version is already
# stored on the selected Target Disk.
if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fileId,
fileVersion)):
# This file is already stored on the given disk.
# Add to the exempt list.
diskExemptList.append(trgDiskInfo.getDiskId())
else:
# OK, this disk should be OK, stop looking for a
# suitable Target Disk.
break
else:
trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), targetDiskId)
slotId = trgDiskInfo.getSlotId()
storageSetId = srvObj.getCfg().getStorageSetFromSlotId(slotId).\
getStorageSetId()
trgDiskInfo.setStorageSetId(storageSetId)
# Don't accept to clone onto the same disk (this would meann overwriting).
if (trgDiskInfo.getDiskId() == diskId):
err = "Source and target files are identical"
msg = "Failed in cloning file with ID: " + fileId +\
"/Version: " + str(fileVersion) +\
" on disk with ID: " + diskId +\
" on host: " + hostId + ". Reason: " + err
raise Exception, msg
# Receive the file into the staging filename.
tmpReqPropsObj = ngamsReqProps.ngamsReqProps()
tmpReqPropsObj.setMimeType(mimeType)
stagingFilename = ngamsHighLevelLib.genStagingFilename(
srvObj.getCfg(), tmpReqPropsObj, trgDiskInfo, fileId)
try:
quickLocation = False
if (reqPropsObj.hasHttpPar("quick")):
quickLocation = int(reqPropsObj.getHttpPar("quick"))
# Receive the data into the Staging File using the urllib.
if (srvObj.getHostId() != hostId):
# Example: http://host:7777/RETRIEVE?disk_id=%s&"
# file_id=id&file_version=1
fileUrl = "http://%s:%s/RETRIEVE?disk_id=%s&file_id=%s&" +\
"file_version=%s"
fileUrl = fileUrl % (ipAddress, str(portNo), diskId, fileId,
str(fileVersion))
# If CLONE?quick specified, we try to retrieve the file via the
# RETRIEVE?quick_location method.
quickFileUrl = fileUrl
if (reqPropsObj.hasHttpPar("quick")):
if (int(reqPropsObj.getHttpPar("quick"))):
quickFileUrl = fileUrl + "&quick_location=1"
# Check if host is suspended, if yes, wake it up.
if (srvObj.getDb().getSrvSuspended(hostId)):
logger.debug("Clone Request - Waking up suspended " +\
"NGAS Host: %s", hostId)
ngamsSrvUtils.wakeUpHost(srvObj, hostId)
else:
# TODO: a time-bomb waiting to explode....
fileUrl = "file:" + mtPt + "/" + filename
logger.debug("Receiving file via URI: %s into staging filename: %s",
fileUrl, stagingFilename)
# We try up to 5 times to retrieve the file in case a problem is
# encountered during cloning.
for attempt in range(5):
try:
if (attempt == 0):
filename, headers = urllib.urlretrieve(
quickFileUrl, stagingFilename)
else:
filename, headers = urllib.urlretrieve(
fileUrl, stagingFilename)
_checkFile(srvObj, srcFileInfo, stagingFilename, headers, True)
# If we get to this point the transfer was (probably) OK.
break
except Exception, e:
rmFile(stagingFilename)
errMsg = "Problem occurred while cloning file "+\
"via URL: " + fileUrl + " - Error: " + str(e)
if (attempt < 4):
errMsg += " - Retrying in 5s ..."
logger.error(errMsg)
time.sleep(0.5)
else:
raise Exception, errMsg
# We simply copy the file into the same destination as the
# source file (but on another disk).
targPathName = os.path.dirname(srcFileInfo.getFilename())
targFilename = os.path.basename(srcFileInfo.getFilename())
complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\
"/" + targPathName)
checkCreatePath(complTargPath)
complFilename = os.path.normpath(complTargPath + "/" + targFilename)
mvTime = mvFile(stagingFilename, complFilename)
ngamsLib.makeFileReadOnly(complFilename)
# Update status for new file in the DB.
newFileInfo = srcFileInfo.clone().setDiskId(trgDiskInfo.getDiskId()).\
setCreationDate(getFileCreationTime(complFilename))
fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(), fileId,
fileVersion)
newFileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Update status for the Target Disk in DB + check if the disk is
# completed.
if (fileExists): mvTime = 0
dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\
setDiskId(trgDiskInfo.getDiskId()).\
setFileExists(fileExists).\
setFileSize(srcFileInfo.getFileSize()).\
setIoTime(mvTime)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj)
ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId())
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, fileId, fileVersion, filename)
# Generate a confirmation log entry.
msg = genLog("NGAMS_INFO_FILE_CLONED",
[fileId, fileVersion, diskId, hostId])
logger.info(msg, extra={'to_syslog': True})
def _cloneExec(srvObj, cloneListDbmName, tmpFilePat, targetDiskId,
reqPropsObj):
"""
See documentation of ngamsCloneCmd._cloneThread(). This function is
merely implemented in order to encapsulate the whole process to be able
to clean up properly when the processing is terminated.
"""
T = TRACE(1)
cloneListDbm = cloneStatusDbm = None
emailNotif = 0
checkChecksum = 1
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1
if (reqPropsObj.hasHttpPar("check")):
checkChecksum = int(reqPropsObj.getHttpPar("check"))
# Open clone list DB.
cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName)
# We have to get the port numbers of the hosts where the files to be
# cloned are stored.
hostInfoDic = {}
cloneListDbm.initKeyPtr()
while (1):
key, fileInfo = cloneListDbm.getNext()
if (not key): break
hostInfoDic[fileInfo[1]] = -1
hostInfoDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
hostInfoDic.keys())
# The cloning loop. Loop over the list of files to clone and generate
# a report with the result.
if (emailNotif):
cloneStatusDbmName = tmpFilePat + "_CLONE_STATUS_DB"
cloneStatusDbm = ngamsDbm.ngamsDbm(cloneStatusDbmName,
cleanUpOnDestr=0,
writePerm=1)
successCloneCount = 0
failedCloneCount = 0
abortCloneLoop = 0
timeAccu = 0.0
key = 0
while (1):
clone_start = time.time()
if (not cloneListDbm.hasKey(str(key))): break
fileInfo = cloneListDbm.get(str(key))
key += 1
# Check if we have permission to run. Otherwise, stop.
if (not srvObj.getThreadRunPermission()): break
fio = fileInfo[0]
mtPt = fileInfo[2]
if (emailNotif):
tmpFileList = ngamsFileList.\
ngamsFileList("FILE_CLONE_STATUS",
"File: " + fio.getFileId() + "/" +\
fio.getDiskId() + "/" +\
str(fio.getFileVersion()))
hostId = fileInfo[1]
text = "Cloning file - File ID: %s/%d, on disk " +\
"with ID: %s on host: %s"
logger.debug(text, fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId)
# We generate a local Staging File and archive this.
stagingFilename = ""
try:
# Check if file is marked as bad.
if (fio.getFileStatus()[0] == "1"):
errMsg = "File marked as bad - skipping!"
raise Exception, errMsg
if (targetDiskId == ""):
# Try to find a disk not hosting already a file with that
# ID + version.
diskExemptList = [fio.getDiskId()]
while (1):
trgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
fio.getFormat(),
1, diskExemptList)
# Check if a file with that ID + version is already
# stored on the selected Target Disk.
if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())):
# This file is already stored on the given disk.
# Add to the exempt list.
diskExemptList.append(trgDiskInfo.getDiskId())
else:
# OK, this disk should be OK, stop looking for a
# suitable Target Disk.
break
else:
try:
trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), targetDiskId)
slotId = trgDiskInfo.getSlotId()
storageSetId = srvObj.getCfg().\
getStorageSetFromSlotId(slotId).\
getStorageSetId()
trgDiskInfo.setStorageSetId(storageSetId)
except Exception:
abortCloneLoop = 1
raise
# We don't accept to clone onto the same disk (this would mean
# overwriting).
if (trgDiskInfo.getDiskId() == fio.getDiskId()):
err = "Source and target files are identical"
msg = "Failed in cloning file with ID: " + fio.getFileId() +\
"/Version: " + str(fio.getFileVersion()) +\
" on disk with ID: " + fio.getDiskId() +\
" on host: " + hostId + ". Reason: " + err
logger.warning(msg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": " + err)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
continue
tmpReqPropsObj = ngamsReqProps.ngamsReqProps()
tmpReqPropsObj.setMimeType(fio.getFormat())
stagingFilename = ngamsHighLevelLib.\
genStagingFilename(srvObj.getCfg(),
tmpReqPropsObj,
trgDiskInfo, fio.getFileId())
# Receive the data into the Staging File using the urllib.
if (srvObj.getHostId() != hostId):
# Example: http://host:7777/RETRIEVE?file_id=id&file_version=1
ipAddress = hostInfoDic[hostId].getIpAddress()
portNo = hostInfoDic[hostId].getSrvPort()
fileUrl = "http://" + ipAddress + ":" + str(portNo) +\
"/RETRIEVE?" + "file_id=" + fio.getFileId() +\
"&file_version=" + str(fio.getFileVersion())
# If a specific Disk ID for the source file is given, append
# this.
if (fio.getDiskId()):
fileUrl += "&disk_id=%s" % fio.getDiskId()
# Check if host is suspended, if yes, wake it up.
if (srvObj.getDb().getSrvSuspended(hostId)):
logger.debug("Clone Request - Waking up suspended " +\
"NGAS Host: %s", hostId)
ngamsSrvUtils.wakeUpHost(srvObj, hostId)
else:
fileUrl = "file:" + mtPt + "/" + fio.getFilename()
logger.debug(
"Receiving file via URI: %s into staging filename: %s",
fileUrl, stagingFilename)
# We try up to 5 times to retrieve the file in case a problem is
# encountered during cloning.
for attempt in range(5):
try:
filename, headers = urllib.urlretrieve(
fileUrl, stagingFilename)
_checkFile(srvObj, fio, stagingFilename, headers,
checkChecksum)
# If we get to this point the transfer was (probably) OK.
break
except Exception, e:
rmFile(stagingFilename)
errMsg = "Problem occurred while cloning file "+\
"via URL: " + fileUrl + " - Error: " + str(e)
if (attempt < 4):
errMsg += " - Retrying in 5s ..."
logger.error(errMsg)
time.sleep(5)
else:
raise Exception, errMsg
# We simply copy the file into the same destination as the
# source file (but on another disk).
targPathName = os.path.dirname(fio.getFilename())
targFilename = os.path.basename(fio.getFilename())
complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\
"/" + targPathName)
checkCreatePath(complTargPath)
complFilename = os.path.normpath(complTargPath + "/" +
targFilename)
mvTime = mvFile(stagingFilename, complFilename)
ngamsLib.makeFileReadOnly(complFilename)
# Update status for new file in the DB.
newFileInfo = fio.clone().setDiskId(trgDiskInfo.getDiskId()).\
setCreationDate(getFileCreationTime(complFilename))
fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())
newFileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Update status for the Target Disk in DB + check if the disk is
# completed.
if (fileExists): mvTime = 0
dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\
setDiskId(trgDiskInfo.getDiskId()).\
setFileExists(fileExists).\
setFileSize(fio.getFileSize()).setIoTime(mvTime)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj)
ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId())
# Update the clone file status list.
if (emailNotif):
tmpFileList.setStatus(NGAMS_SUCCESS)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
tmpFileList.addFileInfoObj(newFileInfo.setTag("TARGET_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
successCloneCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
diskId = trgDiskInfo.getDiskId()
fileId = fio.getFileId()
fileVer = fio.getFileVersion()
filename = fio.getFilename()
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, fileId, fileVer, filename)
# Generate a confirmation log entry.
cloneTime = time.time() - clone_start
timeAccu += cloneTime
msg = genLog("NGAMS_INFO_FILE_CLONED", [
fio.getFileId(),
fio.getFileVersion(),
fio.getDiskId(), hostId
])
msg = msg + ". Time: %.3fs. Total time: %.3fs." %\
(cloneTime, timeAccu)
logger.info(msg, extra={'to_syslog': True})
except Exception, e:
cloneTime = time.time() - clone_start
timeAccu += cloneTime
errMsg = genLog("NGAMS_ER_FILE_CLONE_FAILED", [
fio.getFileId(),
fio.getFileVersion(),
fio.getDiskId(), hostId,
str(e)
])
if (abortCloneLoop):
logger.error(errMsg, extra={'to_syslog': True})
thread.exit()
else:
logger.warning(errMsg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": Error: " + errMsg)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
# Delete Staging File if already created.
if ((stagingFilename != "") and (os.path.exists(stagingFilename))):
rmFile(stagingFilename)
def _dumpFileInfo(srvObj, disks_to_check, tmpFilePat, stopEvt):
"""
Function that dumps the information about the files. One DBM is created
per disk. This is named:
<Mount Root Point>/cache/DATA-CHECK-THREAD_QUEUE_<Disk ID>.bsddb
If problems are found for a file, these are stored in DBM files named:
<Mount Root Point>/cache/DATA-CHECK-THREAD_ERRORS_<Disk ID>.bsddb
The function handles the DBM files in the following way:
1. Check for each DBM file found, if this disk is still in the system.
If not, the Queue and Error DBM files are removed.
2. Go through the list of disks in the system. If they don't have
the two DBM files listed above, these are initialized. The file
information for all the files stored on the disk is dumped into
the Queue DBM file. Only files marked to be ignored are not dumped.
3. Finally, build up a DBM with references to all files found
on this system
srvObj: Reference to server object (ngamsServer).
tmpFilePat: Pattern for temporary files (string).
Returns: Void.
"""
T = TRACE()
cacheDir = os.path.join(srvObj.getCfg().getRootDirectory(),
NGAMS_CACHE_DIR)
checkCreatePath(os.path.normpath(cacheDir))
###########################################################################
# Loop over the Queue/Error DBM files found, check if the disk is
# still in the system/scheduled for checking.
###########################################################################
logger.debug("Loop over/check existing Queue/Error DBM Files ...")
dbmFileList = glob.glob(cacheDir + "/" + NGAMS_DATA_CHECK_THR +\
"_QUEUE_*.bsddb")
dbmObjDic = {}
for dbmFile in dbmFileList:
_stopDataCheckThr(stopEvt)
diskId = dbmFile.split("_")[-1].split(".")[0]
if diskId not in disks_to_check:
filePat = "%s/%s*%s.bsddb" % (cacheDir, NGAMS_DATA_CHECK_THR,
diskId)
rmFile(filePat)
else:
# Add references to Queue/Error DBM.
queueDbmFile = "%s/%s_QUEUE_%s.bsddb" %\
(cacheDir, NGAMS_DATA_CHECK_THR, diskId)
queueDbm = ngamsDbm.ngamsDbm(queueDbmFile, 0, 1)
errorDbmFile = "%s/%s_ERRORS_%s.bsddb" %\
(cacheDir, NGAMS_DATA_CHECK_THR, diskId)
errorDbm = ngamsDbm.ngamsDbm(errorDbmFile, 0, 1)
dbmObjDic[diskId] = (queueDbm, errorDbm)
logger.debug("Looped over/checked existing Queue/Error DBM Files")
###########################################################################
###########################################################################
# Loop over the disks mounted in this system and check if they have a
# Queue/Error DBM file. In case the DBM files are not available, create
# these.
###########################################################################
logger.debug("Create DBM files for disks to be checked ...")
startDbFileRd = time.time()
for diskId in disks_to_check.keys():
_stopDataCheckThr(stopEvt)
if diskId in dbmObjDic:
continue
# The disk is ripe for checking but still has no Queue/Error DBM
# DBs allocated.
queueDbmFile = "%s/%s_QUEUE_%s.bsddb" %\
(cacheDir, NGAMS_DATA_CHECK_THR, diskId)
tmpQueueDbmFile = tmpFilePat + "_" + os.path.basename(queueDbmFile)
queueDbm = ngamsDbm.ngamsDbm(tmpQueueDbmFile, 0, 1)
# Now, retrieve the files on the given disk, and store the info
# in the Queue DBM file.
files = srvObj.getDb().getFileSummary1(diskIds=[diskId],
ignore=0,
fileStatus=[],
lowLimIngestDate=None,
order=0)
for fileInfo in files:
fileId = fileInfo[ngamsDbCore.SUM1_FILE_ID]
fileVer = fileInfo[ngamsDbCore.SUM1_VERSION]
fileKey = ngamsLib.genFileKey(None, fileId, fileVer)
queueDbm.add(fileKey, fileInfo)
queueDbm.sync()
# Rename DCC Queue DBM from the temporary to the final name.
mvFile(tmpQueueDbmFile, queueDbmFile)
queueDbm = ngamsDbm.ngamsDbm(queueDbmFile, 0, 1)
# Create Error DBM + add these in the DBM Dictionary for the disk.
errorDbmFile = "%s/%s_ERRORS_%s.bsddb" %\
(cacheDir, NGAMS_DATA_CHECK_THR, diskId)
errorDbm = ngamsDbm.ngamsDbm(errorDbmFile, 0, 1)
dbmObjDic[diskId] = (queueDbm, errorDbm)
_stopDataCheckThr(stopEvt)
logger.debug("Queried info for files to be checked from DB. Time: %.3fs",
time.time() - startDbFileRd)
logger.debug("Checked that disks scheduled for checking have DBM files")
###########################################################################
# These are all files recursively found on the disks
# Later on we check whether they are registered or not, and check them (or not)
start = time.time()
files_on_disk = collect_files_on_disk(stopEvt, disks_to_check)
end = time.time()
logger.debug("Collected references to %d files on disks in %.3f [s]",
len(files_on_disk), end - start)
# Don't take these into account
logger.debug("Retrieving information about files to be ignored ...")
files = srvObj.getDb().getFileSummarySpuriousFiles1(srvObj.getHostId(),
fetch_size=1000)
for fileInfo in files:
if (fileInfo[ngamsDbCore.SUM1_FILE_IGNORE]):
filename = os.path.\
normpath(fileInfo[ngamsDbCore.SUM1_MT_PT] + "/" +\
fileInfo[ngamsDbCore.SUM1_FILENAME])
if filename in files_on_disk:
del files_on_disk[filename]
logger.debug("Retrieved information about files to be ignored")
###########################################################################
###########################################################################
# Initialize the statistics parameters for the checking.
###########################################################################
logger.debug("Initialize the statistics for the checking cycle ...")
amountMb = 0.0
noOfFiles = 0
for diskId in disks_to_check.keys():
queueDbm = dbmObjDic[diskId][0]
#################################################################################################
#jagonzal: Replace looping aproach to avoid exceptions coming from the next() method underneath
# when iterating at the end of the table that are prone to corrupt the hash table object
#queueDbm.initKeyPtr()
#while (1):
# fileKey, fileInfo = queueDbm.getNext()
# if (not fileKey): break
for fileKey, dbVal in queueDbm.iteritems():
# jagonzal: We need to reformat the values and skip administrative elements #################
if (str(fileKey).find("__") != -1): continue
fileInfo = cPickle.loads(dbVal)
#############################################################################################
noOfFiles += 1
amountMb += float(fileInfo[ngamsDbCore.SUM1_FILE_SIZE]) / 1048576.0
#################################################################################################
stats = _initFileCheckStatus(srvObj, amountMb, noOfFiles)
###########################################################################
return files_on_disk, dbmObjDic, stats
def _cloneExec(srvObj,
cloneListDbmName,
tmpFilePat,
targetDiskId,
reqPropsObj):
"""
See documentation of ngamsCloneCmd._cloneThread(). This function is
merely implemented in order to encapsulate the whole process to be able
to clean up properly when the processing is terminated.
"""
cloneStatusDbm = None
emailNotif = 0
checkChecksum = 1
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")): emailNotif = 1
if (reqPropsObj.hasHttpPar("check")):
checkChecksum = int(reqPropsObj.getHttpPar("check"))
# Open clone list DB.
cloneListDbm = ngamsDbm.ngamsDbm(cloneListDbmName)
# We have to get the port numbers of the hosts where the files to be
# cloned are stored.
hostInfoDic = {}
cloneListDbm.initKeyPtr()
while (1):
key, fileInfo = cloneListDbm.getNext()
if (not key): break
hostInfoDic[fileInfo[1]] = -1
hostInfoDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
hostInfoDic.keys())
# The cloning loop. Loop over the list of files to clone and generate
# a report with the result.
if (emailNotif):
cloneStatusDbmName = tmpFilePat + "_CLONE_STATUS_DB"
cloneStatusDbm = ngamsDbm.ngamsDbm(cloneStatusDbmName,
cleanUpOnDestr = 0, writePerm = 1)
successCloneCount = 0
failedCloneCount = 0
abortCloneLoop = 0
timeAccu = 0.0
key = 0
while (1):
clone_start = time.time()
if (not cloneListDbm.hasKey(str(key))): break
fileInfo = cloneListDbm.get(str(key))
key += 1
# Check if we have permission to run. Otherwise, stop.
if (not srvObj.run_async_commands): break
fio = fileInfo[0]
mtPt = fileInfo[2]
if (emailNotif):
tmpFileList = ngamsFileList.\
ngamsFileList("FILE_CLONE_STATUS",
"File: " + fio.getFileId() + "/" +\
fio.getDiskId() + "/" +\
str(fio.getFileVersion()))
hostId = fileInfo[1]
text = "Cloning file - File ID: %s/%d, on disk " +\
"with ID: %s on host: %s"
logger.debug(text, fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId)
# We generate a local Staging File and archive this.
stagingFilename = ""
try:
# Check if file is marked as bad.
if (fio.getFileStatus()[0] == "1"):
errMsg = "File marked as bad - skipping!"
raise Exception(errMsg)
if (targetDiskId == ""):
# Try to find a disk not hosting already a file with that
# ID + version.
diskExemptList = [fio.getDiskId()]
while (1):
trgDiskInfo = ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
fio.getFormat(),
1, diskExemptList)
# Check if a file with that ID + version is already
# stored on the selected Target Disk.
if (srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())):
# This file is already stored on the given disk.
# Add to the exempt list.
diskExemptList.append(trgDiskInfo.getDiskId())
else:
# OK, this disk should be OK, stop looking for a
# suitable Target Disk.
break
else:
try:
trgDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), targetDiskId)
slotId = trgDiskInfo.getSlotId()
storageSetId = srvObj.getCfg().\
getStorageSetFromSlotId(slotId).\
getStorageSetId()
trgDiskInfo.setStorageSetId(storageSetId)
except Exception:
abortCloneLoop = 1
raise
# We don't accept to clone onto the same disk (this would mean
# overwriting).
if (trgDiskInfo.getDiskId() == fio.getDiskId()):
err = "Source and target files are identical"
msg = "Failed in cloning file with ID: " + fio.getFileId() +\
"/Version: " + str(fio.getFileVersion()) +\
" on disk with ID: " + fio.getDiskId() +\
" on host: " + hostId + ". Reason: " + err
logger.warning(msg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": " + err)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
continue
tmpReqPropsObj = ngamsReqProps.ngamsReqProps()
tmpReqPropsObj.setMimeType(fio.getFormat())
stagingFilename = ngamsHighLevelLib.\
genStagingFilename(srvObj.getCfg(),
tmpReqPropsObj,
trgDiskInfo, fio.getFileId())
# Receive the data into the Staging File using the urllib.
if (srvObj.getHostId() != hostId):
# Example: http://host:7777/RETRIEVE?file_id=id&file_version=1
ipAddress = hostInfoDic[hostId].getIpAddress()
portNo = hostInfoDic[hostId].getSrvPort()
fileUrl = "http://" + ipAddress + ":" + str(portNo) +\
"/RETRIEVE?" + "file_id=" + fio.getFileId() +\
"&file_version=" + str(fio.getFileVersion())
# If a specific Disk ID for the source file is given, append
# this.
if (fio.getDiskId()):
fileUrl += "&disk_id=%s" % fio.getDiskId()
# Check if host is suspended, if yes, wake it up.
if (srvObj.getDb().getSrvSuspended(hostId)):
logger.debug("Clone Request - Waking up suspended " +\
"NGAS Host: %s", hostId)
ngamsSrvUtils.wakeUpHost(srvObj, hostId)
else:
fileUrl = "file:" + mtPt + "/" + fio.getFilename()
logger.debug("Receiving file via URI: %s into staging filename: %s",
fileUrl, stagingFilename)
# We try up to 5 times to retrieve the file in case a problem is
# encountered during cloning.
for attempt in range(5):
try:
filename, headers = urlrequest.urlretrieve(fileUrl, stagingFilename)
_checkFile(srvObj, fio, stagingFilename, headers,
checkChecksum)
# If we get to this point the transfer was (probably) OK.
break
except Exception as e:
rmFile(stagingFilename)
errMsg = "Problem occurred while cloning file "+\
"via URL: " + fileUrl + " - Error: " + str(e)
if (attempt < 4):
errMsg += " - Retrying in 5s ..."
logger.error(errMsg)
time.sleep(5)
else:
raise Exception(errMsg)
# We simply copy the file into the same destination as the
# source file (but on another disk).
targPathName = os.path.dirname(fio.getFilename())
targFilename = os.path.basename(fio.getFilename())
complTargPath = os.path.normpath(trgDiskInfo.getMountPoint() +\
"/" + targPathName)
checkCreatePath(complTargPath)
complFilename = os.path.normpath(complTargPath + "/"+targFilename)
mvTime = mvFile(stagingFilename, complFilename)
ngamsLib.makeFileReadOnly(complFilename)
# Update status for new file in the DB.
newFileInfo = fio.clone().setDiskId(trgDiskInfo.getDiskId()).\
setCreationDate(getFileCreationTime(complFilename))
fileExists = srvObj.getDb().fileInDb(trgDiskInfo.getDiskId(),
fio.getFileId(),
fio.getFileVersion())
newFileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Update status for the Target Disk in DB + check if the disk is
# completed.
if (fileExists): mvTime = 0
dummyDapiStatObj = ngamsDapiStatus.ngamsDapiStatus().\
setDiskId(trgDiskInfo.getDiskId()).\
setFileExists(fileExists).\
setFileSize(fio.getFileSize()).setIoTime(mvTime)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), dummyDapiStatObj)
ngamsArchiveUtils.checkDiskSpace(srvObj, trgDiskInfo.getDiskId())
# Update the clone file status list.
if (emailNotif):
tmpFileList.setStatus(NGAMS_SUCCESS)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
tmpFileList.addFileInfoObj(newFileInfo.setTag("TARGET_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
successCloneCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
diskId = trgDiskInfo.getDiskId()
fileId = fio.getFileId()
fileVer = fio.getFileVersion()
filename = fio.getFilename()
ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId,
fileId, fileVer,
filename)
# Generate a confirmation log entry.
cloneTime = time.time() - clone_start
timeAccu += cloneTime
msg = genLog("NGAMS_INFO_FILE_CLONED",
[fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId])
msg = msg + ". Time: %.3fs. Total time: %.3fs." %\
(cloneTime, timeAccu)
logger.info(msg, extra={'to_syslog': True})
except Exception as e:
cloneTime = time.time() - clone_start
timeAccu += cloneTime
errMsg = genLog("NGAMS_ER_FILE_CLONE_FAILED",
[fio.getFileId(), fio.getFileVersion(),
fio.getDiskId(), hostId, str(e)])
if (abortCloneLoop):
logger.error(errMsg, extra={'to_syslog': True})
return
else:
logger.warning(errMsg)
if (emailNotif):
tmpFileList.setStatus(NGAMS_FAILURE + ": Error: " + errMsg)
tmpFileList.addFileInfoObj(fio.setTag("SOURCE_FILE"))
cloneStatusDbm.addIncKey(tmpFileList)
failedCloneCount += 1
# Delete Staging File if already created.
if ((stagingFilename != "") and (os.path.exists(stagingFilename))):
rmFile(stagingFilename)
# Calculate time statistics.
if (reqPropsObj):
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj.\
incActualCount(1), timeAccu)
# Final update of the Request Status.
if (reqPropsObj):
complPercent = (100.0 * (float(reqPropsObj.getActualCount()) /
float(reqPropsObj.getExpectedCount())))
reqPropsObj.setCompletionPercent(complPercent, 1)
reqPropsObj.setCompletionTime(1)
srvObj.updateRequestDb(reqPropsObj)
# Send Clone Report with list of files cloned to a possible
# requestor(select) of this.
totFiles = (successCloneCount + failedCloneCount)
if (emailNotif):
xmlStat = 0
# TODO: Generation of XML status report is disabled since we cannot
# handle for the moment XML documents with 1000s of elements.
if (xmlStat):
cloneStatusFileList = ngamsFileList.\
ngamsFileList("FILE_CLONING_STATUS_REPORT",
"File Cloning Status Report")
fileCount = 0
while (fileCount < cloneStatusDbm.getCount()):
tmpFileList = cloneStatusDbm.get(str(fileCount))
cloneStatusFileList.addFileListObj(tmpFileList)
# Make overall status.
cloneStatusFileList.setStatus("SUCCESS: " +\
str(successCloneCount) +\
", FAILURE: " +\
str(failedCloneCount) +\
", NOT DONE: " +\
str(len(cloneStatusFileList) -\
successCloneCount -\
failedCloneCount))
status = srvObj.genStatus(NGAMS_SUCCESS,
"CLONE command status report").\
addFileList(cloneStatusFileList)
statRep = status.genXmlDoc(0, 0, 0, 1, 0)
statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep)
mimeType = NGAMS_XML_MT
else:
# Generate a 'simple' ASCII report.
statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
fo = open(statRep, "w")
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
else:
diskId = "-----"
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
else:
fileId = "-----"
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = reqPropsObj.getHttpPar("file_version")
else:
fileVersion = "-----"
tmpFormat = "CLONE STATUS REPORT:\n\n" +\
"==Summary:\n\n" +\
"Date: %s\n" +\
"NGAS Host: %s\n" +\
"Disk ID: %s\n" +\
"File ID: %s\n" +\
"File Version: %s\n" +\
"Total Number of Files: %d\n" +\
"Number of Cloned Files: %d\n" +\
"Number of Failed Files: %d\n" +\
"Total processing time (s): %.3f\n" +\
"Handling time per file (s): %.3f\n\n" +\
"==File List:\n\n"
fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), diskId, fileId,
str(fileVersion), totFiles,
successCloneCount, failedCloneCount,
timeAccu, (timeAccu / totFiles)))
tmpFormat = "%-70s %-70s %-7s\n"
fo.write(tmpFormat % ("Source File", "Target File", "Status"))
fo.write(tmpFormat % (70 * "-", 70 * "-", 7 * "-"))
key = 1
while (1):
if (not cloneStatusDbm.hasKey(str(key))): break
tmpFileList = cloneStatusDbm.get(str(key))
key += 1
srcFileObj = tmpFileList.getFileInfoObjList()[0]
srcFile = "%s/%s/%d" % (srcFileObj.getDiskId(),
srcFileObj.getFileId(),
srcFileObj.getFileVersion())
if (tmpFileList.getStatus() == NGAMS_SUCCESS):
trgFileObj = tmpFileList.getFileInfoObjList()[1]
trgFile = "%s/%s/%d" % (trgFileObj.getDiskId(),
trgFileObj.getFileId(),
trgFileObj.getFileVersion())
else:
trgFile = "-----"
fo.write(tmpFormat % (srcFile,trgFile,tmpFileList.getStatus()))
fo.write(149 * "-")
fo.write("\n\n==END\n")
fo.close()
mimeType = NGAMS_TEXT_MT
# Send out the status report.
emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",")
attachmentName = "CloneStatusReport"
if (reqPropsObj.hasHttpPar("disk_id")):
attachmentName += "-" + reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
attachmentName += "-" + reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
attachmentName += "-" + reqPropsObj.getHttpPar("file_version")
ngamsNotification.notify(srvObj.host_id, srvObj.cfg, NGAMS_NOTIF_INFO,
"CLONE STATUS REPORT", statRep, recList=emailAdrList, force=1,
contentType=mimeType, attachmentName=attachmentName)
del cloneStatusDbm
rmFile(cloneStatusDbmName + "*")
rmFile(statRep)
if (cloneListDbm): del cloneListDbm
rmFile(cloneListDbmName + "*")
logger.info("_cloneExec(). Total time: %.3fs. Average time per file: %.3fs.",
timeAccu, (timeAccu / totFiles))
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle the CARCHIVE Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: (fileId, filePath) tuple.
"""
T = TRACE()
# Is this NG/AMS permitted to handle Archive Requests?
logger.debug("Is this NG/AMS permitted to handle Archive Requests?")
if (not srvObj.getCfg().getAllowArchiveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"])
raise Exception(errMsg)
srvObj.checkSetState("Archive Request", [NGAMS_ONLINE_STATE],
[NGAMS_IDLE_SUBSTATE, NGAMS_BUSY_SUBSTATE],
NGAMS_ONLINE_STATE,
NGAMS_BUSY_SUBSTATE,
updateDb=False)
if httpRef.command != 'POST':
raise Exception("Only POST allowed for CARCHIVE")
# Get mime-type (try to guess if not provided as an HTTP parameter).
logger.debug(
"Get mime-type (try to guess if not provided as an HTTP parameter).")
if (reqPropsObj.getMimeType() == ""):
mimeType = ngamsHighLevelLib.\
determineMimeType(srvObj.getCfg(), reqPropsObj.getFileUri())
reqPropsObj.setMimeType(mimeType)
else:
mimeType = reqPropsObj.getMimeType()
# Determine the target volume, ignoring the stream concept.
logger.debug("Determine the target volume, ignoring the stream concept.")
targDiskInfo = ngamsArchiveUtils._random_target_volume(srvObj)
if (targDiskInfo == None):
errMsg = "No disk volumes are available for ingesting any files."
raise Exception(errMsg)
reqPropsObj.setTargDiskInfo(targDiskInfo)
# Generate staging filename.
logger.debug("Generate staging filename from URI: %s",
reqPropsObj.getFileUri())
if (reqPropsObj.getFileUri().find("file_id=") >= 0):
file_id = reqPropsObj.getFileUri().split("file_id=")[1]
baseName = os.path.basename(file_id)
else:
baseName = os.path.basename(reqPropsObj.getFileUri())
stgFilename = os.path.join("/", targDiskInfo.getMountPoint(),
NGAMS_STAGING_DIR,
genUniqueId() + "___" + baseName)
logger.debug("Staging filename is: %s", stgFilename)
reqPropsObj.setStagingFilename(stgFilename)
# Retrieve file contents (from URL, archive pull, or by storing the body
# of the HTTP request, archive push).
stagingInfo = saveInStagingFile(srvObj.getCfg(), reqPropsObj, httpRef,
stgFilename, targDiskInfo)
ioTime = stagingInfo[0]
rootContainer = stagingInfo[1]
fileDataList = stagingInfo[2]
ingestRate = stagingInfo[3]
reqPropsObj.incIoTime(ioTime)
createContainers(rootContainer, None, srvObj)
parDic = {}
ngamsGenDapi.handlePars(reqPropsObj, parDic)
diskInfo = reqPropsObj.getTargDiskInfo()
# Generate file information.
logger.debug("Generate file information")
dateDir = toiso8601(fmt=FMT_DATE_ONLY)
resDapiList = []
containerSizes = {}
for item in fileDataList:
container = item[0]
filepath = item[1]
crc = item[2]
containerId = str(container.getContainerId())
basename = os.path.basename(filepath)
fileId = basename
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfo(srvObj.getDb(),
srvObj.getCfg(),
reqPropsObj, diskInfo,
filepath,
fileId,
basename, [dateDir])
complFilename, relFilename = ngamsGenDapi.checkForDblExt(
complFilename, relFilename)
# Keep track of the total size of the container
uncomprSize = ngamsPlugInApi.getFileSize(filepath)
if containerId not in containerSizes:
containerSizes[containerId] = 0
containerSizes[containerId] += uncomprSize
mimeType = reqPropsObj.getMimeType()
compression = "NONE"
archFileSize = ngamsPlugInApi.getFileSize(filepath)
resDapi = ngamsPlugInApi.genDapiSuccessStat(
diskInfo.getDiskId(), relFilename, fileId, fileVersion, mimeType,
archFileSize, uncomprSize, compression, relPath,
diskInfo.getSlotId(), fileExists, complFilename)
# Move file to final destination.
logger.debug("Moving file to final destination")
ioTime = mvFile(filepath, resDapi.getCompleteFilename())
reqPropsObj.incIoTime(ioTime)
# Get crc info
checksumPlugIn = "StreamCrc32"
checksum = str(crc)
# Get source file version
# e.g.: http://ngas03.hq.eso.org:7778/RETRIEVE?file_version=1&file_id=X90/X962a4/X1
logger.debug("Get file version")
file_version = resDapi.getFileVersion()
if reqPropsObj.getFileUri().count("file_version"):
file_version = int(
(reqPropsObj.getFileUri().split("file_version=")[1]
).split("&")[0])
# Check/generate remaining file info + update in DB.
logger.debug("Creating db entry")
creDate = getFileCreationTime(resDapi.getCompleteFilename())
fileInfo = ngamsFileInfo.ngamsFileInfo().\
setDiskId(resDapi.getDiskId()).\
setFilename(resDapi.getRelFilename()).\
setFileId(resDapi.getFileId()).\
setFileVersion(file_version).\
setFormat(resDapi.getFormat()).\
setFileSize(resDapi.getFileSize()).\
setUncompressedFileSize(resDapi.getUncomprSize()).\
setCompression(resDapi.getCompression()).\
setIngestionDate(time.time()).\
setChecksum(checksum).setChecksumPlugIn(checksumPlugIn).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDate).\
setIoTime(reqPropsObj.getIoTime())
fileInfo.write(srvObj.getHostId(), srvObj.getDb())
# Add the file to the container
srvObj.getDb().addFileToContainer(containerId, resDapi.getFileId(),
True)
# Update the container sizes
for contSizeInfo in containerSizes.items():
srvObj.getDb().setContainerSize(contSizeInfo[0], contSizeInfo[1])
# Inform the caching service about the new file.
logger.debug("Inform the caching service about the new file.")
if (srvObj.getCachingActive()):
diskId = resDapi.getDiskId()
fileId = resDapi.getFileId()
fileVersion = file_version
filename = resDapi.getRelFilename()
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, fileId, fileVersion, filename)
# Update disk info in NGAS Disks.
logger.debug("Update disk info in NGAS Disks.")
srvObj.getDb().updateDiskInfo(resDapi.getFileSize(),
resDapi.getDiskId())
resDapiList.append(resDapi)
# Check if the disk is completed.
# We use an approximate extimate for the remaning disk space to avoid
# to read the DB.
logger.debug("Check available space in disk")
availSpace = getDiskSpaceAvail(targDiskInfo.getMountPoint(), smart=False)
if (availSpace < srvObj.getCfg().getFreeSpaceDiskChangeMb()):
targDiskInfo.setCompleted(1).setCompletionDate(time.time())
targDiskInfo.write(srvObj.getDb())
# Request after-math ...
srvObj.setSubState(NGAMS_IDLE_SUBSTATE)
msg = "Successfully handled Archive Pull Request for data file " +\
"with URI: " + reqPropsObj.getSafeFileUri()
logger.info(msg)
httpRef.send_ingest_status(msg, targDiskInfo)
for resDapi in resDapiList:
# Trigger Subscription Thread. This is a special version for MWA, in which we simply swapped MIRRARCHIVE and QARCHIVE
# [email protected]
logger.debug("triggering SubscriptionThread for file %s",
resDapi.getFileId())
srvObj.addSubscriptionInfo(
[(resDapi.getFileId(), resDapi.getFileVersion())], [])
srvObj.triggerSubscriptionThread()
def _registerExec(srvObj,
fileListDbmName,
tmpFilePat,
diskInfoDic,
reqPropsObj=None):
"""
Register the files listed in the File List DBM (ngamsDbm), which
match the mime-type(s) either specified in the 'mimeType' parameter,
or if this is not specified, which match all the mime-types specified
in the configuration file.
When the registration procedure has been executed, the function sends
an Email Notification message indicating which files were registered
if the HTTP parameter 'notif_email' is given.
The functions creates a File Info Objects per file handled and
writes this in a temporary file, which is a DBM file. The
keys in this DB is simply the file number in the sequence of files
handled, pointing to a pickled ngamsFileInfo object.
Each of the File Info Objects indicates if the file was registered or not.
This is done by setting the tag of the File Info Object to one of the
following values:
REGISTERED: The file was successfully registered in the NGAS DB
FAILED: The file was selected for registration but could not
be properly registered because of inconsistencies.
The status will be of the format: 'FAILED[: <reason>]'.
REJECTED: A file found under the specified path directory was
not accepted for cloning, usually because the mime-type
was not correct. The status will be of the format:
'REJECTED[: <reason>]'.
Note, that registration is comparable to archiving of files. For that
reason a DAPI must be provided for each type of file that should be
registered. If this is not fullfilled, the file registration will
fail.
Only files stored on one of the NGAS disks configured in the
configuration file, are considered. Files stored in other locations
are ignored.
srvObj: Instance of NG/AMS Server object (ngamsServer).
fileListDbmName: Name of a DBM containing the information
about the files to be registered. Each element in the
list is referred to by a key, which is a number.
These points to a pickled list for each file containing
the following information:
[<Filename>, <Disk ID>, <Mime-Type>]
The information for each disk concerned is also contained
in the DB referred to by its Disk ID and by its mount
point. The data is a pickled instance of the ngamsDiskInfo
class (string).
tmpFilePat: Pattern for temporary files used during the registration
process (string).
diskInfoDic: Dictionary with Disk IDs as keys pointing to the info
about the disk (dictionary/ngamsDiskInfo).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
Returns: Void.
"""
emailNotif = 0
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")):
emailNotif = 1
# Create the temporary BSD DB to contain the information for the
# Email Notification Message.
if (emailNotif):
regDbmName = tmpFilePat + "_NOTIF_EMAIL"
regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm=1)
# Open the DBM containing the list of files to (possibly) register.
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1)
# Want to parse files in alphabetical order.
# TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting.
tmpFileList = tmpFilePat + "_FILE_LIST"
rmFile(tmpFileList)
with open(tmpFileList, "wb") as fo:
fileListDbm.initKeyPtr()
while (1):
dbmKey, fileInfo = fileListDbm.getNext()
if (not dbmKey): break
fo.write(dbmKey + b"\n")
sortFileList = tmpFilePat + "_SORT_FILE_LIST"
rmFile(sortFileList)
shellCmd = "sort %s > %s" % (tmpFileList, sortFileList)
stat, out, err = ngamsCore.execCmd(shellCmd)
if (stat != 0):
raise Exception("Error executing command: %s. Error: %s, %s" %\
(shellCmd, str(out), str(err)))
rmFile(tmpFileList)
# Go through each file in the list, check if the mime-type is among the
# ones, which apply for registration. If yes try to register the file
# by invoking the corresponding DAPI on the file.
fileRegCount = 0
fileFailCount = 0
fileRejectCount = 0
regTimeAccu = 0.0
fileCount = 0
fo = open(sortFileList)
run = 1
while (run):
reg_start = time.time()
dbmKey = fo.readline()
if (dbmKey.strip() == ""):
run = 0
continue
fileInfo = fileListDbm.get(dbmKey[0:-1])
filename = fileInfo[0]
diskId = fileInfo[1]
mimeType = fileInfo[2]
# Register the file. Check first, that exactly this file is
# not already registered. In case it is, the file will be rejected.
regPi = srvObj.getCfg().register_plugins[mimeType]
logger.debug("Plugin found for %s: %s", mimeType, regPi)
params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars)
tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\
setMimeType(mimeType).\
setStagingFilename(filename).\
setTargDiskInfo(diskInfoDic[diskId]).\
setHttpMethod(NGAMS_HTTP_GET).\
setCmd(NGAMS_REGISTER_CMD).\
setSize(os.path.getsize(filename)).\
setFileUri(filename).\
setNoReplication(1)
tmpFileObj = ngamsFileInfo.ngamsFileInfo()
try:
# Invoke Registration Plug-In.
piName = regPi.name
plugInMethod = loadPlugInEntryPoint(piName)
piRes = plugInMethod(srvObj, tmpReqPropsObj, params)
del tmpReqPropsObj
# Check if this file is already registered on this disk. In case
# yes, it is not registered again.
files = srvObj.db.getFileSummary1(srvObj.getHostId(),
[piRes.getDiskId()],
[piRes.getFileId()])
fileRegistered = 0
for tmpFileInfo in files:
tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT]
tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME]
tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\
tmpFilename)
if (tmpComplFilename == filename):
fileRegistered = 1
break
if (fileRegistered):
fileRejectCount += 1
tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\
"and path: %s is already registered on disk " +\
"with Disk ID: %s"
tmpMsg = tmpMsgForm % (piRes.getFileId(),
piRes.getFileVersion(), filename,
piRes.getDiskId())
logger.warning(tmpMsg + ". File is not registered again.")
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(tmpMsg)
regDbm.addIncKey(tmpFileObj)
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(
srvObj, reqPropsObj, regTimeAccu)
regTimeAccu += time.time() - reg_start
fileCount += 1
continue
# Calculate checksum. We maintain the old name for backwards
# compatibility
crc_variant = srvObj.cfg.getCRCVariant()
if crc_variant == ngamsFileUtils.CHECKSUM_CRC32_INCONSISTENT:
crc_variant = 'ngamsGenCrc32'
checksum = ngamsFileUtils.get_checksum(65536, filename,
crc_variant) or ''
# Move file and update information about file in the NGAS DB.
mvFile(filename, piRes.getCompleteFilename())
ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum,
crc_variant)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes)
ngamsLib.makeFileReadOnly(piRes.getCompleteFilename())
if (emailNotif):
uncomprSize = piRes.getUncomprSize()
ingestDate = time.time()
creDateSecs = getFileCreationTime(filename)
tmpFileObj.\
setDiskId(diskId).\
setFilename(filename).\
setFileId(piRes.getFileId()).\
setFileVersion(piRes.getFileVersion()).\
setFormat(piRes.getFormat()).\
setFileSize(piRes.getFileSize()).\
setUncompressedFileSize(uncomprSize).\
setCompression(piRes.getCompression()).\
setIngestionDate(ingestDate).\
setIgnore(0).\
setChecksum(checksum).\
setChecksumPlugIn(crc_variant).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDateSecs).\
setTag("REGISTERED")
fileRegCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
fileVer = fio.getFileVersion()
ngamsCacheControlThread.addEntryNewFilesDbm(
srvObj, diskId, piRes.getFileId(), fileVer, filename)
# Generate a confirmation log entry.
msg = genLog("NGAMS_INFO_FILE_REGISTERED", [
filename,
piRes.getFileId(),
piRes.getFileVersion(),
piRes.getFormat()
])
time.sleep(0.005)
regTime = time.time() - reg_start
msg = msg + ". Time: %.3fs." % (regTime)
logger.info(msg, extra={'to_syslog': 1})
except Exception as e:
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)])
logger.error(errMsg)
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(errMsg)
fileFailCount += 1
regTime = time.time() - reg_start
# TODO (rtobar, 2016-01): Why don't we raise an exception here?
# Otherwise the command appears as successful on the
# client-side
# Add the file information in the registration report.
if (emailNotif): regDbm.addIncKey(tmpFileObj)
# Update request status time information.
regTimeAccu += regTime
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
fileCount += 1
fo.close()
rmFile(sortFileList)
if (emailNotif): regDbm.sync()
del fileListDbm
rmFile(fileListDbmName + "*")
# Final update of the Request Status.
if (reqPropsObj):
if (reqPropsObj.getExpectedCount() and reqPropsObj.getActualCount()):
complPercent = (100.0 * (float(reqPropsObj.getActualCount()) /
float(reqPropsObj.getExpectedCount())))
else:
complPercent = 100.0
reqPropsObj.setCompletionPercent(complPercent, 1)
reqPropsObj.setCompletionTime(1)
srvObj.updateRequestDb(reqPropsObj)
# Send Register Report with list of files cloned to a possible
# requestor(select) of this.
if (emailNotif):
xmlStat = 0
if (xmlStat):
# Create an instance of the File List Class, used to store the
# Registration Report.
regStat = ngamsFileList.\
ngamsFileList("FILE_REGISTRATION_STATUS",
"Status report for file " +\
"registration")
# Loop over the file objects in the BSD DB and add these
# in the status object.
regDbm.initKeyPtr()
while (1):
key, tmpFileObj = regDbm.getNext()
if (not key): break
regStat.addFileInfoObj(tmpFileObj)
# Set overall status of registration procedure.
regStat.setStatus("Files Found: " + str(fileCount + 1) + ", "+\
"Files Registered: " + str(fileRegCount) +\
", " +\
"Files Failed: " + str(fileFailCount) + ", " +\
"Files Rejected: " + str(fileRejectCount))
status = srvObj.genStatus(NGAMS_SUCCESS,
"REGISTER command status report").\
addFileList(regStat)
statRep = status.genXmlDoc()
statRep = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, statRep)
mimeType = NGAMS_XML_MT
else:
# Generate a 'simple' ASCII report.
statRep = tmpFilePat + "_NOTIF_EMAIL.txt"
fo = open(statRep, "w")
if (reqPropsObj):
path = reqPropsObj.getHttpPar("path")
else:
path = "-----"
if (fileCount):
timePerFile = (regTimeAccu / fileCount)
else:
timePerFile = 0
tmpFormat = "REGISTER STATUS REPORT:\n\n" +\
"==Summary:\n\n" +\
"Date: %s\n" +\
"NGAS Host: %s\n" +\
"Search Path: %s\n" +\
"Total Number of Files: %d\n" +\
"Number of Registered Files: %d\n" +\
"Number of Failed Files: %d\n" +\
"Number of Rejected Files: %d\n" +\
"Total processing time (s): %.3f\n" +\
"Handling time per file (s): %.3f\n\n" +\
"==File List:\n\n"
fo.write(tmpFormat % (toiso8601(), srvObj.getHostId(), path,
fileCount, fileRegCount, fileFailCount,
fileRejectCount, regTimeAccu, timePerFile))
tmpFormat = "%-80s %-32s %-3s %-10s\n"
fo.write(tmpFormat %\
("Filename", "File ID", "Ver", "Status"))
fo.write(tmpFormat % (80 * "-", 32 * "-", 3 * "-", 10 * "-"))
regDbm.initKeyPtr()
while (1):
key, tmpFileObj = regDbm.getNext()
if (not key): break
mtPt = diskInfoDic[tmpFileObj.getDiskId()].getMountPoint()
filename = os.path.normpath(mtPt + "/" +\
tmpFileObj.getFilename())
line = tmpFormat %\
(filename, tmpFileObj.getFileId(),
str(tmpFileObj.getFileVersion()),
tmpFileObj.getTag())
fo.write(line)
fo.write(128 * "-")
fo.write("\n\n==END\n")
fo.close()
mimeType = NGAMS_TEXT_MT
# Send out the status report.
emailAdrList = reqPropsObj.getHttpPar("notif_email").split(",")
attachmentName = "RegisterStatusReport"
if (reqPropsObj.hasHttpPar("path")):
attachmentName += "-" + reqPropsObj.getHttpPar("path").\
replace("/", "_")
ngamsNotification.notify(srvObj.host_id,
srvObj.cfg,
NGAMS_NOTIF_INFO,
"REGISTER STATUS REPORT",
statRep,
recList=emailAdrList,
force=1,
contentType=mimeType,
attachmentName=attachmentName)
del regDbm
rmFile(regDbmName + "*")
rmFile(statRep)
# Generate final status log + exit.
if (fileCount > 0):
timePerFile = (regTimeAccu / fileCount)
else:
timePerFile = 0.0
msg = "Registration procedure finished processing Register Request - " + \
"terminating. Files handled: %d. Total time: %.3fs. " + \
"Average time per file: %.3fs."
logger.debug(msg, fileCount, regTimeAccu, timePerFile)
def test_VolumeDir_01(self):
"""
Synopsis:
Grouping of data volumes under the Volume Dir in the NGAS Root Dir.
Description:
See ngamsArchiveCmdTest.test_VolumeDir_01().
This tests verifies that the files archived into this structure can
be received.
Expected Result:
When the server goes Online, it should accept the given directory
structure and it should be possible to archive files into this
structureand subsequently to receive them.
Test Steps:
- Create the volume dirs from an existing structure.
- Start server with configuration specifying the Volumes Dir in which
all volumes will be hosted.
- Archive a FITS file.
- Retrieve the files and check that they are OK.
Remarks:
...
"""
# Create basic structure.
ngasRootDir = "/tmp/ngamsTest/NGAS/"
rmFile(ngasRootDir)
checkCreatePath(ngasRootDir)
subprocess.check_call(['tar', 'zxf', 'src/volumes_dir.tar.gz'])
mvFile('volumes', ngasRootDir)
# Create configuration, start server.
cwd = os.getcwd()
configFile = os.path.normpath(cwd +
"/src/ngamsCfg_VolumeDirectory.xml")
self.prepExtSrv(delDirs=0, cfgFile=configFile)
client = sendPclCmd()
# Archive a file.
stat = client.archive("src/SmallFile.fits")
# dpallot: this will always fail on the mac as the tar sizes are different
# to the hard coded test results in the old file:
# ngamsRetrieveCmdTest_test_VolumeDir_01_01_ref
#tmpStatFile = saveInFile(None, filterDbStatus1(stat.dumpBuf()))
#refStatFile = "ref/ngamsRetrieveCmdTest_test_VolumeDir_01_01_ref"
#self.checkFilesEq(refStatFile, tmpStatFile, "Incorrect status " +\
# "message from NG/AMS Server")
self.assertEquals(stat.getStatus(), 'SUCCESS')
# Check that the target files have been archived in their
# appropriate locations.
trgFile = "tmp/test_VolumeDir_01_tmp"
refFile = "src/SmallFile.fits"
outFilePath = "tmp/SmallFile.fits"
stat = client.retrieve("TEST.2001-05-08T15:25:00.123",
targetFile=trgFile)
self.assertEqual(NGAMS_SUCCESS, stat.getStatus(), stat.getMessage())
# unzip the the file and diff against original
unzip(trgFile, outFilePath)
self.checkFilesEq(outFilePath, refFile, "Retrieved file incorrect")
def _registerExec(srvObj,
fileListDbmName,
tmpFilePat,
diskInfoDic,
reqPropsObj = None):
"""
Register the files listed in the File List DBM (ngamsDbm), which
match the mime-type(s) either specified in the 'mimeType' parameter,
or if this is not specified, which match all the mime-types specified
in the configuration file.
When the registration procedure has been executed, the function sends
an Email Notification message indicating which files were registered
if the HTTP parameter 'notif_email' is given.
The functions creates a File Info Objects per file handled and
writes this in a temporary file, which is a DBM file. The
keys in this DB is simply the file number in the sequence of files
handled, pointing to a pickled ngamsFileInfo object.
Each of the File Info Objects indicates if the file was registered or not.
This is done by setting the tag of the File Info Object to one of the
following values:
REGISTERED: The file was successfully registered in the NGAS DB
FAILED: The file was selected for registration but could not
be properly registered because of inconsistencies.
The status will be of the format: 'FAILED[: <reason>]'.
REJECTED: A file found under the specified path directory was
not accepted for cloning, usually because the mime-type
was not correct. The status will be of the format:
'REJECTED[: <reason>]'.
Note, that registration is comparable to archiving of files. For that
reason a DAPI must be provided for each type of file that should be
registered. If this is not fullfilled, the file registration will
fail.
Only files stored on one of the NGAS disks configured in the
configuration file, are considered. Files stored in other locations
are ignored.
srvObj: Instance of NG/AMS Server object (ngamsServer).
fileListDbmName: Name of a DBM containing the information
about the files to be registered. Each element in the
list is referred to by a key, which is a number.
These points to a pickled list for each file containing
the following information:
[<Filename>, <Disk ID>, <Mime-Type>]
The information for each disk concerned is also contained
in the DB referred to by its Disk ID and by its mount
point. The data is a pickled instance of the ngamsDiskInfo
class (string).
tmpFilePat: Pattern for temporary files used during the registration
process (string).
diskInfoDic: Dictionary with Disk IDs as keys pointing to the info
about the disk (dictionary/ngamsDiskInfo).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
Returns: Void.
"""
T = TRACE()
emailNotif = 0
if (reqPropsObj):
if (reqPropsObj.hasHttpPar("notif_email")):
emailNotif = 1
# Create the temporary BSD DB to contain the information for the
# Email Notification Message.
if (emailNotif):
regDbmName = tmpFilePat + "_NOTIF_EMAIL"
regDbm = ngamsDbm.ngamsDbm(regDbmName, writePerm = 1)
# Open the DBM containing the list of files to (possibly) register.
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm = 1)
# Want to parse files in alphabetical order.
# TODO: Portatibility issue. Try to avoid UNIX shell commands for sorting.
tmpFileList = tmpFilePat + "_FILE_LIST"
rmFile(tmpFileList)
fo = open(tmpFileList, "w")
fileListDbm.initKeyPtr()
while (1):
dbmKey, fileInfo = fileListDbm.getNext()
if (not dbmKey): break
fo.write(dbmKey + "\n")
fo.close()
sortFileList = tmpFilePat + "_SORT_FILE_LIST"
rmFile(sortFileList)
shellCmd = "sort %s > %s" % (tmpFileList, sortFileList)
stat, out = commands.getstatusoutput(shellCmd)
if (stat != 0):
raise Exception, "Error executing command: %s. Error: %s" %\
(shellCmd, str(out))
rmFile(tmpFileList)
# Go through each file in the list, check if the mime-type is among the
# ones, which apply for registration. If yes try to register the file
# by invoking the corresponding DAPI on the file.
checksumPlugIn = srvObj.getCfg().getChecksumPlugIn().strip()
fileRegCount = 0
fileFailCount = 0
fileRejectCount = 0
regTimeAccu = 0.0
fileCount = 0
fo = open(sortFileList)
run = 1
while (run):
reg_start = time.time()
dbmKey = fo.readline()
if (dbmKey.strip() == ""):
run = 0
continue
fileInfo = fileListDbm.get(dbmKey[0:-1])
filename = fileInfo[0]
diskId = fileInfo[1]
mimeType = fileInfo[2]
# Register the file. Check first, that exactly this file is
# not already registered. In case it is, the file will be rejected.
regPi = srvObj.getCfg().register_plugins[mimeType]
logger.debug("Plugin found for %s: %s", mimeType, regPi)
params = ngamsPlugInApi.parseRawPlugInPars(regPi.pars)
tmpReqPropsObj = ngamsReqProps.ngamsReqProps().\
setMimeType(mimeType).\
setStagingFilename(filename).\
setTargDiskInfo(diskInfoDic[diskId]).\
setHttpMethod(NGAMS_HTTP_GET).\
setCmd(NGAMS_REGISTER_CMD).\
setSize(os.path.getsize(filename)).\
setFileUri(filename).\
setNoReplication(1)
tmpFileObj = ngamsFileInfo.ngamsFileInfo()
try:
# Invoke Registration Plug-In.
piName = regPi.name
plugInMethod = loadPlugInEntryPoint(piName)
piRes = plugInMethod(srvObj, tmpReqPropsObj, params)
del tmpReqPropsObj
# Check if this file is already registered on this disk. In case
# yes, it is not registered again.
files = srvObj.db.getFileSummary1(srvObj.getHostId(), [piRes.getDiskId()],
[piRes.getFileId()])
fileRegistered = 0
for tmpFileInfo in files:
tmpMtPt = tmpFileInfo[ngamsDbCore.SUM1_MT_PT]
tmpFilename = tmpFileInfo[ngamsDbCore.SUM1_FILENAME]
tmpComplFilename = os.path.normpath(tmpMtPt + "/" +\
tmpFilename)
if (tmpComplFilename == filename):
fileRegistered = 1
break
if (fileRegistered):
fileRejectCount += 1
tmpMsgForm = "REJECTED: File with File ID/Version: %s/%d " +\
"and path: %s is already registered on disk " +\
"with Disk ID: %s"
tmpMsg = tmpMsgForm % (piRes.getFileId(),
piRes.getFileVersion(), filename,
piRes.getDiskId())
logger.warning(tmpMsg + ". File is not registered again.")
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(tmpMsg)
regDbm.addIncKey(tmpFileObj)
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
regTimeAccu += time.time() - reg_start
fileCount += 1
continue
# Calculate checksum (if plug-in specified).
if (checksumPlugIn != ""):
logger.debug("Invoking Checksum Plug-In: %s to handle file: %s",
checksumPlugIn, filename)
plugInMethod = loadPlugInEntryPoint(checksumPlugIn)
checksum = plugInMethod(srvObj, filename, 0)
else:
checksum = ""
# Move file and update information about file in the NGAS DB.
mvFile(filename, piRes.getCompleteFilename())
ngamsArchiveUtils.updateFileInfoDb(srvObj, piRes, checksum,
checksumPlugIn)
ngamsDiskUtils.updateDiskStatusDb(srvObj.getDb(), piRes)
ngamsLib.makeFileReadOnly(piRes.getCompleteFilename())
if (emailNotif):
uncomprSize = piRes.getUncomprSize()
ingestDate = time.time()
creDateSecs = getFileCreationTime(filename)
tmpFileObj.\
setDiskId(diskId).\
setFilename(filename).\
setFileId(piRes.getFileId()).\
setFileVersion(piRes.getFileVersion()).\
setFormat(piRes.getFormat()).\
setFileSize(piRes.getFileSize()).\
setUncompressedFileSize(uncomprSize).\
setCompression(piRes.getCompression()).\
setIngestionDate(ingestDate).\
setIgnore(0).\
setChecksum(checksum).\
setChecksumPlugIn(checksumPlugIn).\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDate(creDateSecs).\
setTag("REGISTERED")
fileRegCount += 1
# If running as a cache archive, update the Cache New Files DBM
# with the information about the new file.
if (srvObj.getCachingActive()):
fileVer = fio.getFileVersion()
ngamsCacheControlThread.addEntryNewFilesDbm(srvObj, diskId,
piRes.getFileId(),
fileVer, filename)
# Generate a confirmation log entry.
msg = genLog("NGAMS_INFO_FILE_REGISTERED",
[filename, piRes.getFileId(), piRes.getFileVersion(),
piRes.getFormat()])
time.sleep(0.005)
regTime = time.time() - reg_start
msg = msg + ". Time: %.3fs." % (regTime)
logger.info(msg, extra={'to_syslog': 1})
except Exception, e:
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [filename, str(e)])
logger.error(errMsg)
if (emailNotif):
tmpFileObj.\
setDiskId(diskId).setFilename(filename).\
setTag(errMsg)
fileFailCount += 1
regTime = time.time() - reg_start
# TODO (rtobar, 2016-01): Why don't we raise an exception here?
# Otherwise the command appears as successful on the
# client-side
# Add the file information in the registration report.
if (emailNotif): regDbm.addIncKey(tmpFileObj)
# Update request status time information.
regTimeAccu += regTime
if (reqPropsObj):
reqPropsObj.incActualCount(1)
ngamsHighLevelLib.stdReqTimeStatUpdate(srvObj, reqPropsObj,
regTimeAccu)
fileCount += 1
def __handleCmd(srvObj, reqPropsObj):
"""
Handle the Mirroring Archive (MIRRARCHIVE) Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).setState
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
Returns: Void.
"""
# Is this NG/AMS permitted to handle Archive Requests?
logger.debug(
"Checking if this NG/AMS permitted to handle Archive Requests?")
if (not srvObj.getCfg().getAllowArchiveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"])
raise Exception(errMsg)
# Generate staging filename.
stgFilename = reqPropsObj.getStagingFilename()
logger.info("staging filename is: %s", stgFilename)
startByte = 0
if (os.path.exists(stgFilename) == 0):
logger.debug('this is a new staging file')
else:
startByte = os.path.getsize(stgFilename)
logger.debug(
'staging file already exists, requesting resumption of download from byte %d',
startByte)
# Set reference in request handle object to the read socket.
try:
# Retrieve file_id and file_version from request proposal
file_id = reqPropsObj.fileinfo['fileId']
file_version = reqPropsObj.fileinfo['fileVersion']
logger.debug("Got file_id=%s and file_version=%s", file_id,
file_version)
# Retrieve file contents (from URL, archive pull, or by storing the body
# of the HTTP request, archive push).
logger.info("Saving in staging file: %s", stgFilename)
stagingInfo = saveInStagingFile(srvObj, srvObj.getCfg(), reqPropsObj,
stgFilename, startByte)
reqPropsObj.incIoTime(stagingInfo[0])
checksumPlugIn = "ngamsGenCrc32"
checksum = stagingInfo[1]
except (ngamsFailedDownloadException.FailedDownloadException,
ngamsFailedDownloadException.PostponeException):
raise
except Exception as e:
if getattr(e, 'errno', 0) == 28:
# we can't resume, otherwise the same host will be used next time
logger.warning(
"ran out of disk space during the download to %s. Marking as FAILURE",
stgFilename)
# TBD automatically mark the volume as completed
# TBD try something more sophisticated with the file - other volumes on the same host?
# ot at least ARCHIVE in another host avoiding the download of WAN again. This is
# particularly important if we just downloaded most of a 300GB file
raise ngamsFailedDownloadException.FailedDownloadException(e)
elif reqPropsObj.getBytesReceived() >= 0:
logger.warning(
"the fetch has already downloaded data. marking as TORESUME")
raise ngamsFailedDownloadException.PostponeException(e)
else:
logger.warning(
3, "no data has been downloaded yet. Marking as FAILURE")
raise ngamsFailedDownloadException.FailedDownloadException(e)
# Invoke DAPI
logger.info("Invoking DAPI")
resDapi = ngamsDAPIMirroring.ngamsGeneric(srvObj, reqPropsObj)
# Move file to final destination.
logger.info("Moving file to final destination: %s",
resDapi.getCompleteFilename())
ioTime = mvFile(reqPropsObj.getStagingFilename(),
resDapi.getCompleteFilename())
reqPropsObj.incIoTime(ioTime)
# Check/generate remaining file info + update in DB.
logger.info("Creating db entry")
creDate = srvObj.getDb().convertTimeStamp(
getFileCreationTime(resDapi.getCompleteFilename()))
sqlUpdate = "update ngas_disks set available_mb = available_mb - {0} / (1024 * 1024), bytes_stored = bytes_stored + {1} " +\
"where disk_id = {2}"
srvObj.getDb().query2(sqlUpdate,
args=(resDapi.getFileSize(), resDapi.getFileSize(),
resDapi.getDiskId()))
ts = srvObj.getDb().convertTimeStamp(time.time())
sqlQuery = "INSERT INTO ngas_files " +\
"(disk_id, file_name, file_id, file_version, " +\
"format, file_size, " +\
"uncompressed_file_size, compression, " +\
"ingestion_date, %s, checksum, " % ('file_ignore' if srvObj.getCfg().getDbUseFileIgnore() else 'ignore') +\
"checksum_plugin, file_status, creation_date) "+\
"VALUES " +\
"({}, {}, {}, {}," +\
" {}, {}," +\
" {}, {}," +\
" {}, 0, {}," +\
" {}, {}, {})"
args = (str(resDapi.getDiskId()), str(resDapi.getRelFilename()), file_id,
file_version, str(resDapi.getFormat()), resDapi.getFileSize(),
resDapi.getUncomprSize(), str(resDapi.getCompression()), ts,
str(checksum), checksumPlugIn, NGAMS_FILE_STATUS_OK, creDate)
logger.info("Will try to insert the file information: %s / %r", sqlQuery,
args)
srvObj.getDb().query2(sqlQuery, args=args)
# Final log message
logger.info(
"Successfully handled Archive Pull Request for data file with URI: %s",
reqPropsObj.getSafeFileUri())
return
# EOF
def archiveFromFile(srvObj,
filename,
noReplication=0,
mimeType=None,
reqPropsObj=None):
"""
Archive a file directly from a file as source.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
filename: Name of file to archive (string).
noReplication: Flag to enable/disable replication (integer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
Returns: Execution result object of DAPI
"""
T = TRACE()
logger.debug("Archiving file: %s", filename)
logger.debug("Mimetype used is %s", mimeType)
if (reqPropsObj):
logger.debug("Request Properties Object given - using this")
reqPropsObjLoc = reqPropsObj
else:
logger.debug("No Request Properties Object given - creating one")
reqPropsObjLoc = ngamsArchiveUtils.ngamsReqProps.ngamsReqProps()
stagingFile = filename
try:
if (mimeType == None):
mimeType = ngamsHighLevelLib.determineMimeType(
srvObj.getCfg(), filename)
archive_start = time.time()
# Prepare dummy ngamsReqProps object (if an object was not given).
if (not reqPropsObj):
reqPropsObjLoc.setMimeType(mimeType)
reqPropsObjLoc.setStagingFilename(filename)
reqPropsObjLoc.setHttpMethod(NGAMS_HTTP_GET)
reqPropsObjLoc.setCmd(NGAMS_ARCHIVE_CMD)
reqPropsObjLoc.setSize(os.path.getsize(filename))
reqPropsObjLoc.setFileUri(NGAMS_HTTP_FILE_URL + filename)
reqPropsObjLoc.setNoReplication(noReplication)
# If no target disk is defined, find one suitable disk.
if (not reqPropsObjLoc.getTargDiskInfo()):
try:
trgDiskInfo = ngamsArchiveUtils.ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
mimeType, 0,
reqSpace=reqPropsObjLoc.getSize())
reqPropsObjLoc.setTargDiskInfo(trgDiskInfo)
# copy the file to the staging area of the target disk
stagingFile = trgDiskInfo.getMountPoint(
) + '/staging/' + os.path.basename(filename)
cpFile(filename, stagingFile)
reqPropsObjLoc.setStagingFilename(stagingFile)
except Exception as e:
errMsg = str(e) + ". Attempting to archive local file: " +\
filename
ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_NO_DISKS,
"NO DISKS AVAILABLE", errMsg)
raise Exception(errMsg)
# Set the log cache to 1 during the handling of the file.
plugIn = srvObj.getMimeTypeDic()[mimeType]
logger.debug("Invoking DAPI: %s to handle file: %s", plugIn,
stagingFile)
plugInMethod = loadPlugInEntryPoint(plugIn)
resMain = plugInMethod(srvObj, reqPropsObjLoc)
# Move the file to final destination.
st = time.time()
mvFile(reqPropsObjLoc.getStagingFilename(),
resMain.getCompleteFilename())
iorate = reqPropsObjLoc.getSize() / (time.time() - st)
ngamsArchiveUtils.postFileRecepHandling(srvObj, reqPropsObjLoc,
resMain, trgDiskInfo)
except Exception as e:
# If another error occurs, than one qualifying for Back-Log
# Buffering the file, we have to log an error.
if (ngamsHighLevelLib.performBackLogBuffering(srvObj.getCfg(),
reqPropsObjLoc, e)):
logger.exception(
"Tried to archive local file %s, keeping original file",
filename)
return [NGAMS_FAILURE, str(e), NGAMS_FAILURE]
else:
logger.exception("Tried to archive local file %s, " + \
"moving it to Bad Files Directory " + \
"-- cannot be handled", filename)
ngamsHighLevelLib.moveFile2BadDir(srvObj.getCfg(), filename)
# Remove pickle file if available.
pickleObjFile = filename + "." + NGAMS_PICKLE_FILE_EXT
if (os.path.exists(pickleObjFile)):
logger.debug("Removing Back-Log Buffer Pickle File: %s",
pickleObjFile)
rmFile(pickleObjFile)
return [NGAMS_FAILURE, str(e), NGAMS_FAILURE]
# If the file was handled successfully, we remove it from the
# Back-Log Buffer Directory unless the local file was a log-file
# in which case we leave the cleanup to the Janitor-Thread.
if stagingFile.find('LOG-ROTATE') > -1:
logger.debug("Successfully archived local file: %s", filename)
else:
logger.debug(
"Successfully archived local file: %s. Removing staging file.",
filename)
rmFile(stagingFile)
rmFile(stagingFile + "." + NGAMS_PICKLE_FILE_EXT)
logger.debug("Archived local file: %s. Time (s): %.3f", filename,
time.time() - archive_start)
return (resMain, trgDiskInfo, iorate)
def __handle_command(ngams_server, request_properties):
"""
Handle the mirroring archive command (MIRRARCHIVE)
:param ngams_server: Reference to NG/AMS server class object (ngamsServer).setState
:param request_properties: Request Property object to keep track of actions done during the request handling
(ngamsReqProps).
"""
# Is this NG/AMS permitted to handle Archive Requests?
logger.debug("Checking if this NG/AMS permitted to handle Archive Requests?")
if not ngams_server.getCfg().getAllowArchiveReq():
error_message = genLog("NGAMS_ER_ILL_REQ", ["Archive"])
raise Exception(error_message)
# Generate staging filename
staging_filename = request_properties.getStagingFilename()
logger.info("Staging filename is: %s", staging_filename)
start_byte = 0
if os.path.exists(staging_filename) == 0:
logger.debug('Staging file %s is a new file', staging_filename)
else:
start_byte = os.path.getsize(staging_filename)
logger.debug('Staging file %s already exists, requesting resumption of download from byte %d',
staging_filename, start_byte)
# Set reference in request handle object to the read socket
try:
# Retrieve file_id and file_version from request proposal
file_id = request_properties.fileinfo['fileId']
file_version = request_properties.fileinfo['fileVersion']
logger.debug("Got file_id=%s and file_version=%s", file_id, file_version)
# Retrieve file contents (from URL, archive pull, or by storing the body of the HTTP request, archive push)
logger.info("Saving contents in staging file: %s", staging_filename)
staging_info = save_in_staging_file(ngams_server, ngams_server.getCfg(), request_properties,
staging_filename, start_byte)
request_properties.incIoTime(staging_info.rtime)
request_properties.incIoTime(staging_info.wtime)
ingest_rate = int(request_properties.getSize() // staging_info.totaltime)
except (ngamsFailedDownloadException.FailedDownloadException, ngamsFailedDownloadException.PostponeException):
raise
except Exception as e:
if getattr(e, 'errno', 0) == 28:
# We cannot resume, otherwise the same host will be used next time
logger.warning("Ran out of disk space during the download to %s. Marking as FAILURE", staging_filename)
# TODO: automatically mark the volume as completed
# TODO: try something more sophisticated with the file - other volumes on the same host?
# Or at least ARCHIVE in another host avoiding the download of WAN again. This is particularly
# important if we just downloaded most of a 300GB file
raise ngamsFailedDownloadException.FailedDownloadException(e)
elif request_properties.getBytesReceived() >= 0:
logger.warning("A fetch has already downloaded data. Marking as TORESUME")
raise ngamsFailedDownloadException.PostponeException(e)
else:
logger.warning("No data has been downloaded yet. Marking as FAILURE")
raise ngamsFailedDownloadException.FailedDownloadException(e)
# Invoke DAPI
logger.info("Invoking DAPI for mirroring")
result_dapi = ngamsDAPIMirroring.ngams_generic(ngams_server, request_properties)
# Move file to final destination
logger.info("Moving file to final destination: %s", result_dapi.getCompleteFilename())
mtime = mvFile(request_properties.getStagingFilename(), result_dapi.getCompleteFilename())
request_properties.incIoTime(mtime)
# Check/generate remaining file info and update in DB
logger.info("Creating DB entry")
creation_date = ngams_server.getDb().convertTimeStamp(getFileCreationTime(result_dapi.getCompleteFilename()))
sql = "update ngas_disks " \
"set available_mb = available_mb - {0} / (1024 * 1024), bytes_stored = bytes_stored + {1} " \
"where disk_id = {2}"
ngams_server.getDb().query2(sql, args=(result_dapi.getFileSize(), result_dapi.getFileSize(),
result_dapi.getDiskId()))
timestamp = ngams_server.getDb().convertTimeStamp(time.time())
io_time = int(request_properties.getIoTime() * 1000)
sql = "insert into ngas_files " \
"(disk_id, file_name, file_id, " \
"file_version, format, file_size, " \
"uncompressed_file_size, compression, ingestion_date, " \
"{:s}, ".format('file_ignore' if ngams_server.getCfg().getDbUseFileIgnore() else 'ignore') +\
"checksum, checksum_plugin, file_status, " \
"creation_date, io_time, ingestion_rate) " \
"VALUES " \
"({}, {}, {}, " \
"{}, {}, {}, " \
"{}, {}, {}, " \
"0, " \
"{}, {}, {}, " \
"{}, {}, {})"
args = (str(result_dapi.getDiskId()), str(result_dapi.getRelFilename()), file_id,
file_version, str(result_dapi.getFormat()), result_dapi.getFileSize(),
result_dapi.getUncomprSize(), str(result_dapi.getCompression()), timestamp,
str(staging_info.crc), staging_info.crcname, NGAMS_FILE_STATUS_OK,
creation_date, io_time, ingest_rate)
logger.info("Will try to insert the file information: %s / %r", sql, args)
try:
ngams_server.getDb().query2(sql, args=args)
except Exception:
# this shouldn't happen, but it can. If we rapidly restart a server then there can be some race condition
# where a thread is already downloading the file and actually finishes it. Meanwhile the main mirroring thread
# shuts down and sets the status to 'ABORTED'. The next iteration starts mirroring it. before the first thread
# sets the status to 'SUCCESS'. It happens so rarely that it's easier to work-around than fix properly.
logger.exception('mirroring error: the file %s has already been registered. Should not have happened, but no damage done.', file_id)
logger.info("Successfully handled Archive Pull Request for data file with URI: %s",
request_properties.getSafeFileUri())
return
def archiveXTimes(filename, number, hostName, port, targetDir, ouputFile,
increment, auth):
"""
Archive the given FITS file X times. Each time the file is archived the
value of the ARCFILE keyword can be incremented to enforce a new
File ID.
filename: FITS file to archive (string).
number: Number of times to archive the FITS file (integer).
hostName: Host name of remote NG/AMS Server. Can also be
a list of servers (string).
port: Port number of remote NG/AMS Server (integer).
targetDir: If specified, the created files are merely dumped in the
given directory (string).
ouputFile: Output log file (string).
increment: If set to 1 the ARCFILE keyword is incremented 1ms for
each archiving to create a new File ID (integer/0|1).
auth: Authentication code (string).
Returns: Void.
"""
if (not targetDir):
# Carry out the archive session.
client = ngamsPClient.ngamsPClient(hostName, port).\
setAuthorization(auth)
# Host list given?
if (hostName.find(":") != -1): client.parseSrvList(hostName)
reqTimer = PccUtTime.Timer()
totTime = 0.0
if (ouputFile): fo = open(ouputFile, "w")
format = "Status of Archive Request number: %d: %s. Time: %.3fs. " +\
"Total time: %.3fs."
if (increment):
tmpFilename = os.path.normpath("/tmp/tmp__" +
os.path.basename(filename))
os.system("cp " + filename + " " + tmpFilename)
commands.getstatusoutput("add_chksum " + tmpFilename)
else:
tmpFilename = filename
ts = None
for n in range(1, (number + 1)):
reqTimer.start()
if (targetDir):
baseName = os.path.basename(tmpFilename)[4:]
tmpTargetFilename = os.path.join(targetDir, "." + str(n) + "_" +\
baseName)
targetFilename = os.path.join(targetDir, str(n) + "_" + baseName)
print "Creating file: %s" % targetFilename
os.system("cp %s %s" % (tmpFilename, tmpTargetFilename))
mvFile(tmpTargetFilename, targetFilename)
reqTime = reqTimer.stop()
totTime += reqTime
else:
print "Archiving file: %s for the time number: %d." %\
(tmpFilename, (n+1))
status = client.archive(tmpFilename)
reqTime = reqTimer.stop()
totTime += reqTime
if (ouputFile):
fo.write("%-4d %.3f\n" % ((n + 1), reqTime))
fo.flush()
print format % ((n + 1), status.getStatus(), reqTime, totTime)
# Increment the ARCFILE keyword.
if (increment):
arcFile = pyfits.getval(tmpFilename, 'ARCFILE')
idx = arcFile.find(".")
insId = arcFile[0:idx]
ts = arcFile[(idx + 1):]
newMjd = PccUtTime.TimeStamp().initFromTimeStamp(ts).getMjd() +\
(0.001 / 86400)
newArcFile = insId + "." +\
PccUtTime.TimeStamp(newMjd).getTimeStamp()
pyfits.setval(tmpFilename, 'ARCFILE', value=newArcFile)
commands.getstatusoutput("add_chksum " + tmpFilename)
if (ouputFile): fo.close()
format = "\nFinished archiving file: %s, %d times. Total time: %.3fs. "+\
"Average time per frame: %.3fs.\n"
print format % (filename, number, totTime, float(totTime / float(number)))
def archiveFromFile(srvObj,
filename,
noReplication=0,
mimeType=None,
reqPropsObj=None):
"""
Archive a file directly from a file as source.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
filename: Name of file to archive (string).
noReplication: Flag to enable/disable replication (integer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
Returns: Execution result object of DAPI
"""
T = TRACE()
logger.debug("Archiving file: %s", filename)
logger.debug("Mimetype used is %s", mimeType)
if (reqPropsObj):
logger.debug("Request Properties Object given - using this")
reqPropsObjLoc = reqPropsObj
else:
logger.debug("No Request Properties Object given - creating one")
reqPropsObjLoc = ngamsArchiveUtils.ngamsReqProps.ngamsReqProps()
stagingFile = filename
try:
if (mimeType == None):
mimeType = ngamsHighLevelLib.determineMimeType(
srvObj.getCfg(), filename)
archive_start = time.time()
# Prepare dummy ngamsReqProps object (if an object was not given).
if (not reqPropsObj):
reqPropsObjLoc.setMimeType(mimeType)
reqPropsObjLoc.setStagingFilename(filename)
reqPropsObjLoc.setHttpMethod(NGAMS_HTTP_GET)
reqPropsObjLoc.setCmd(NGAMS_ARCHIVE_CMD)
reqPropsObjLoc.setSize(os.path.getsize(filename))
reqPropsObjLoc.setFileUri(NGAMS_HTTP_FILE_URL + filename)
reqPropsObjLoc.setNoReplication(noReplication)
# If no target disk is defined, find one suitable disk.
if (not reqPropsObjLoc.getTargDiskInfo()):
try:
trgDiskInfo = ngamsArchiveUtils.ngamsDiskUtils.\
findTargetDisk(srvObj.getHostId(),
srvObj.getDb(), srvObj.getCfg(),
mimeType, 0,
reqSpace=reqPropsObjLoc.getSize())
reqPropsObjLoc.setTargDiskInfo(trgDiskInfo)
# copy the file to the staging area of the target disk
stagingFile = trgDiskInfo.getMountPoint(
) + '/staging/' + os.path.basename(filename)
cpFile(filename, stagingFile)
reqPropsObjLoc.setStagingFilename(stagingFile)
except Exception, e:
errMsg = str(e) + ". Attempting to archive local file: " +\
filename
ngamsPlugInApi.notify(srvObj, NGAMS_NOTIF_NO_DISKS,
"NO DISKS AVAILABLE", errMsg)
raise Exception, errMsg
# Set the log cache to 1 during the handling of the file.
plugIn = srvObj.getMimeTypeDic()[mimeType]
logger.debug("Invoking DAPI: %s to handle file: %s", plugIn,
stagingFile)
plugInMethod = loadPlugInEntryPoint(plugIn)
resMain = plugInMethod(srvObj, reqPropsObjLoc)
# Move the file to final destination.
st = time.time()
mvFile(reqPropsObjLoc.getStagingFilename(),
resMain.getCompleteFilename())
iorate = reqPropsObjLoc.getSize() / (time.time() - st)
ngamsArchiveUtils.postFileRecepHandling(srvObj, reqPropsObjLoc,
resMain, trgDiskInfo)
