def remDisk(srvObj, reqPropsObj, diskId, execute):
"""
Select a disk for removal and remove the information if so specified
from the NGAS DB and delete all information on the disk.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
diskId: ID of disk. Complete ID must be specified. I.e., no
wildcards are handled (string).
execute: If set to 1 the information about the disk will be deleted.
Otherwise only the information about the disk selected for
deletion will be queried (integer/0|1).
Returns: Status object contained information about disk
selected for deletion/deleted (ngamsStatus).
"""
tmpFilePat = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(),
"REMDISK_CMD")
try:
status = _remDisk(srvObj, reqPropsObj, diskId, execute, tmpFilePat)
return status
finally:
rmFile(tmpFilePat + "*")
def _handleFileList(srvObj, reqPropsObj, httpRef):
"""
Handle STATUS?file_list... Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: The File List ID allocated for this request (string).
"""
T = TRACE()
# Should a lower limit for the ingestion date be taken into account.
fromIngDate = None
if (reqPropsObj.hasHttpPar("from_ingestion_date")):
tmpTromIngDate = reqPropsObj.getHttpPar("from_ingestion_date")
fromIngDate = fromiso8601(tmpTromIngDate)
# Handle the unique flag. If this is specified, only information for unique
# File ID/Version pairs are returned.
unique = False
if (reqPropsObj.hasHttpPar("unique")):
unique = int(reqPropsObj.getHttpPar("unique"))
# Dump the file information needed.
fileListId = genUniqueId()
dbmBaseName = STATUS_FILE_LIST_DBM_TAG % fileListId
fileInfoDbmName = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(),
dbmBaseName)
# Dump the file info from the DB. Deal with uniqueness quickly
# (instead of using a new file like before)
try:
fileInfoDbm = ngamsDbm.ngamsDbm(fileInfoDbmName, 0, 1)
fileCount = 1
unique_files = set()
for f in srvObj.db.files_in_host(srvObj.getHostId(),
from_date=fromIngDate):
if unique:
key = str('%s_%d' % (f[2], f[3]))
if key in unique_files:
continue
else:
key = str(fileCount)
fileInfoDbm.add(key, f)
fileCount += 1
except Exception as e:
rmFile(fileInfoDbmName)
msg = "Problem generating file list for STATUS Command. " +\
"Parameters: from_ingestion_date=%s. Error: %s" %\
(str(fromIngDate), str(e))
raise Exception(msg)
return fileListId
def remFile(srvObj, reqPropsObj, diskId, fileId, fileVersion, execute):
"""
Function used to delete information about files in the NGAS DB and to
delete files on the disks. Only local files are considered.
The information selected for deletion is selected applying the
following criterias:
o diskId!="", fileId=="", fileVersion==-1:
All files on the given disk are taken.
o diskId!="", fileId!="", fileVersion==-1:
All files with the given File ID on the disk with the given ID
will be selected. No specific file version will be taken into account.
o diskId!="", fileId!="", fileVersion!=-1:
The referenced file with the given File ID and File Version on the
given ID is selected (if this exists).
o diskId=="", fileId!="", fileVersion==-1:
Not allowed.
o diskId=="", fileId!="", fileVersion!=-1:
Not allowed.
o diskId=="", fileId=="", fileVersion!=-1:
Not allowed.
o diskId!="", fileId=="", fileVersion!=-1:
Not allowed
The function will not carry out the actual deletion of the selected
files if the 'execute' parameter is set to 0. In this case a report
with the items selected for deletion will be generated.
NOTE: No files are selected for deletion, unless there are at least three
copies of the file (File ID + File Version), which apparently are
accessible. In addition, these three copies have to be distributed
on three different disks (storage media).
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
diskId: ID of disk. It is not possible to specify a pattern,
must be exact ID (string).
fileId: ID of file. It is possible to specify a file pattern
using wild cards (string).
fileVersion: Version of file to delete. If not specified, all
files matching the Disk ID + File ID will be selected.
Otherwise it is possible to specify a single file for
deletion (integer).
execute: If set to 0, only a report of the information that has
been selected for deletion is generated (integer/0|1).
Returns: Status object with a list of disks and corresponding
files deleted (ngamsStatus).
"""
T = TRACE()
tmpFilePat = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(),
"REMFILE_CMD")
try:
status = _remFile(srvObj, reqPropsObj, diskId, fileId, fileVersion,
execute, tmpFilePat)
return status
finally:
rmFile(tmpFilePat + "*")
def clone(srvObj,
diskId,
fileId,
fileVersion,
targetDiskId="",
reqPropsObj=None,
httpRef=None):
"""
Carry out the cloning. The conditions for carrying out the cloning
are as follows:
o diskId="", fileId!="", fileVersion=-1:
Clone one file with the given ID. Latest version of the file is taken.
o diskId!="", fileId!="", fileVersion=-1:
Clone one file stored on the given disk. Latest version on that
disk is taken.
o diskId="", fileId!="", fileVersion!=-1:
Clone all files found with the given File Version. Storage location
(Disk ID) is not taken into account.
o diskId!="", fileId!="", fileVersion!=-1:
Clone one file on the given disk with the given File Version.
o diskId!="", fileId="", fileVersion=-1:
Clone all files from the disk with the given ID.
o diskId!="", fileId="", fileVersion!=-1:
Clone all files with the given File Version from the disk with
the ID given.
o diskId="", fileId="", fileVersion!=-1:
Illegal. Not accepted to clone arbitrarily files given by only the
File Version.
o diskId="", fileId="", fileVersion=-1:
Illegal. No files specified.
srvObj: Reference to instance of Server Object (ngamsServer).
diskId: ID of disk hosting file(s) to clone (string).
fileId: ID of file to clone (string).
fileVersion: Version of file(s) to clone (integer).
targetDiskId: ID of disk to where the files cloned should be written
(string).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
tmpFilePat = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(), "CLONE_CMD")
try:
_clone(srvObj, diskId, fileId, fileVersion, targetDiskId, reqPropsObj,
httpRef, tmpFilePat)
except Exception:
rmFile(tmpFilePat + "*")
raise
def register(srvObj, path, mimeType="", reqPropsObj=None, httpRef=None):
"""
Function to generate a list of candidate files to register, and to
launch a thread that actually carries out the registration.
When the possible candidate files haven been selected, a reply is sent
back to the requestor if the 'httpRef' object is given.
srvObj: Instance of NG/AMS Server object (ngamsServer).
path: The path name, which is used as starting point for
the searching for files (string).
mimeType: Comma separated list of mime-types, which should be
considered for registration (string).
reqPropsObj: If an NG/AMS Request Properties Object is given, the
Request Status will be updated as the request is carried
out (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
# Check if the given path or file exists.
if (not os.path.exists(path)):
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED",
[path, "Non-existing file or path."])
raise Exception(errMsg)
# Check if the given path/file is on one of the NGAS Disks.
foundPath = 0
for slotId in srvObj.getDiskDic().keys():
do = srvObj.getDiskDic()[slotId]
if (path.find(do.getMountPoint()) == 0):
foundPath = 1
break
if (not foundPath):
errMsg = genLog("NGAMS_ER_FILE_REG_FAILED", [
path, "File or path specified is not located on an "
"NGAS Disk."
])
raise Exception(errMsg)
# Create file pattern for temporary files.
tmpFilePat = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(),
"REGISTER_CMD")
# Generate dictionary with mime-types to accept.
mimeTypeDic = {}
if (mimeType != ""):
for mt in mimeType.split(","):
try:
mimeTypeDic[mt] = srvObj.getMimeTypeDic()[mt]
except:
errMsg = genLog("NGAMS_ER_REQ_HANDLING",
["Mime-type specified: " + mt + " " +\
"in connection with REGISTER command " +\
"does not have a DAPI defined"])
raise Exception(errMsg)
else:
mimeTypeDic = srvObj.getMimeTypeDic()
# From the Disk Dictionary, generate a dictionary with mappings from
# Disk ID + from Mount Point to the disk information for the disk.
diskInfoDic = {}
mtPt2DiskInfo = {}
for slotId in srvObj.getDiskDic().keys():
if (not srvObj.getCfg().getSlotIdDefined(slotId)): continue
diskId = srvObj.getDiskDic()[slotId].getDiskId()
mtPt = srvObj.getDiskDic()[slotId].getMountPoint()
tmpDiskInfo = ngamsDiskInfo.ngamsDiskInfo().\
read(srvObj.getDb(), diskId)
diskInfoDic[diskId] = tmpDiskInfo
mtPt2DiskInfo[mtPt] = tmpDiskInfo
# Generate a list with all files found under the specified path, which
# are candidates for being registered.
fileListDbmName = tmpFilePat + "_FILE_LIST"
rmFile(fileListDbmName + "*")
fileListDbm = ngamsDbm.ngamsDbm(fileListDbmName, writePerm=1)
mimeTypeMappings = srvObj.getCfg().getMimeTypeMappings()
tmpGlobFile = tmpFilePat + "_FILE_GLOB.glob"
fileCount = 0
searchPath = os.path.normpath(path)
foundVolume = False
for mtPt in mtPt2DiskInfo.keys():
mtPt2 = mtPt
if (mtPt[-1] != "/"): mtPt2 += "/"
if (searchPath.find(mtPt2) == 0):
foundVolume = True
if os.path.isdir(searchPath):
for root, dirs, files in os.walk(searchPath):
for nextFile in files:
if nextFile not in \
[NGAMS_DISK_INFO, \
NGAMS_VOLUME_ID_FILE, \
NGAMS_VOLUME_INFO_FILE]:
mimeType = ngamsLib.detMimeType(
mimeTypeMappings, nextFile, 0)
tmpFileInfo = [
os.path.join(root, nextFile),
mtPt2DiskInfo[mtPt].getDiskId(), mimeType
]
fileListDbm.add(os.path.join(root, nextFile),
tmpFileInfo)
elif os.path.isfile(
searchPath): # the path is actually pointing to a file
nextFile = os.path.basename(searchPath)
root = os.path.dirname(searchPath)
if nextFile not in \
[NGAMS_DISK_INFO, \
NGAMS_VOLUME_ID_FILE, \
NGAMS_VOLUME_INFO_FILE]:
mimeType = ngamsLib.detMimeType(mimeTypeMappings, nextFile,
0)
tmpFileInfo = [
os.path.join(root, nextFile),
mtPt2DiskInfo[mtPt].getDiskId(), mimeType
]
fileListDbm.add(os.path.join(root, nextFile), tmpFileInfo)
if foundVolume: break
# pattern = ""
# # TODO: Portatibility issue. The usage of UNIX commands should be
# # avoided if possible.
# while (1):
# # Use a shell commands here to avoid building up maybe huge lists in
# # memory/in the Python interpreter. A better way could maybe be found
# # avoiding to invoke a shell command.
# rmFile(tmpGlobFile)
# searchPath = os.path.normpath(path + pattern)
# tmpCmd = "find " + searchPath + " -maxdepth 1 > " + tmpGlobFile
# stat, out = commands.getstatusoutput(tmpCmd)
# if (stat != 0): break
# fo = open(tmpGlobFile)
# while (1):
# nextFile = fo.readline().strip()
# if (nextFile == ""): break
# if (nextFile.find("/" + NGAMS_DISK_INFO) != -1): continue
# if (nextFile.find("/" + NGAMS_VOLUME_ID_FILE) != -1): continue
# if (nextFile.find("/" + NGAMS_VOLUME_INFO_FILE) != -1): continue
# if (os.path.isfile(nextFile)):
# nextFile = os.path.normpath(nextFile)
#
# # Find the host disk. Only files located on mounted NGAS
# # disks are considered.
# for mtPt in mtPt2DiskInfo.keys():
# mtPt2 = mtPt
# if (mtPt[-1] != "/"): mtPt2 += "/"
# if (nextFile.find(mtPt2) == 0):
# info(4,"Found candidate file for registering: " +\
# nextFile)
# # Take only files with a Registration Plug-In defined.
# mimeType = ngamsLib.detMimeType(mimeTypeMappings,
# nextFile, 0)
# regPi = srvObj.getCfg().getRegPiFromMimeType(mimeType)
# if (regPi != None):
# tmpFileInfo = [nextFile,
# mtPt2DiskInfo[mtPt].getDiskId(),
# mimeType]
# fileListDbm.add(nextFile, tmpFileInfo)
# break
# fo.close()
# pattern += "/*"
# rmFile(tmpGlobFile)
fileListDbm.sync()
del fileListDbm
# Send intermediate reply if the HTTP Reference object is given.
is_async = 'async' in reqPropsObj and int(reqPropsObj['async'])
if httpRef and is_async:
logger.debug("REGISTER command accepted - generating immediate " +\
"confimation reply to REGISTER command")
# Update the request status in the Request Properties Object.
reqPropsObj.\
setExpectedCount(fileCount).\
setActualCount(0).setCompletionPercent(0)
srvObj.updateRequestDb(reqPropsObj)
status = srvObj.genStatus(NGAMS_SUCCESS,
"Accepted REGISTER command for execution").\
setReqStatFromReqPropsObj(reqPropsObj).\
setActualCount(0)
# Launch the register thread or run the command in foreground if wait=1
if is_async:
args = (srvObj, fileListDbmName, tmpFilePat, diskInfoDic, reqPropsObj,
None)
thrName = NGAMS_REGISTER_THR + threading.current_thread().getName()
regThread = threading.Thread(None, _registerThread, thrName, args)
regThread.setDaemon(0)
regThread.start()
else:
# Carry out the REGISTER Command (directly in this thread) and send
# reply when this is done.
_registerExec(srvObj, fileListDbmName, tmpFilePat, diskInfoDic,
reqPropsObj)
msg = "Successfully handled command REGISTER"
logger.debug(msg)
status = srvObj.genStatus(NGAMS_SUCCESS, msg).\
setReqStatFromReqPropsObj(reqPropsObj).setActualCount(0)
# Send reply if possible.
if (httpRef):
xmlStat = status.genXmlDoc(0, 0, 0, 1, 0)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT)
