def getControllers():
"""
Query the controllers available, and return this in a list.
Returns: List with available 3ware controllers (list).
"""
T = TRACE()
cmd = "sudo /usr/local/sbin/tw_cli info"
stat, out = commands.getstatusoutput(cmd)
if (stat):
raise Exception, "Error invoking 3ware Command Line Tool: " + str(out)
contList = []
for line in out.split("\n"):
line = line.strip()
if (line):
if (line.find("Controller") == 0):
# "Controller 1: 8506-4LP (4)"
contNo = line.split(" ")[1].split(":")[0]
contList.append(contNo)
elif (line[0] == "c"):
contNo = int(line.split(" ")[0][1:])
contList.append(contNo)
contList.sort()
return contList
def _getRefCounts():
"""
Return information about all object allocated and the number of
references pointing to each object. This can be used to check a running
server for memory (object) leaks.
Taken from: http://www.nightmare.com/medusa/memory-leaks.html.
Returns:
"""
T = TRACE()
d = {}
sys.modules
# Collect all classes
for m in sys.modules.values():
for sym in dir(m):
o = getattr(m, sym)
if type(o) is types.ClassType:
d[o] = sys.getrefcount(o)
# sort by refcount
pairs = map(lambda x: (x[1], x[0]), d.items())
pairs.sort()
pairs.reverse()
return pairs
def prepFile(reqPropsObj, parDic):
"""
Prepare the file. If it is compressed, decompress it into a temporary
filename.
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
parDic: Dictionary with parameters for the DAPI. This is generated
with ngamsPlugInApi.parseDapiPlugInPars() (Dictionary).
Returns: Tuple containing:
(<DP ID>, <Date Obs. Night>, <Compr. Ext.>) (tuple).
"""
T = TRACE()
# If the file is already compressed, we have to decompress it.
comprExt = ''
compression = parDic["compression"]
if compression and 'gzip' in compression:
comprExt = 'gz'
tmpFn = reqPropsObj.getStagingFilename()
if tmpFn.lower().endswith('.gz'):
newFn = os.path.splitext(tmpFn)[0]
logger.debug("Decompressing file using gzip: %s", tmpFn)
subprocess.check_call(['gunzip', '-f', tmpFn], shell=False)
logger.debug("Decompression success: %s", newFn)
reqPropsObj.setStagingFilename(newFn)
checkFitsFileSize(reqPropsObj.getStagingFilename())
dpIdInfo = getDpIdInfo(reqPropsObj.getStagingFilename())
return dpIdInfo[1], dpIdInfo[2], comprExt
def getFitsKeys(fitsFile, keyList):
"""
Get a FITS keyword from a FITS file. A dictionary is returned whereby
the keys in the keyword list are the dictionary keys and the value
the elements that these refer to.
fitsFile: Filename of FITS file (string).
keyList: Tuple of keys for which to extract values (tuple).
Returns: Dictionary with the values extracted of the format:
{<key 1>: [<val hdr 0>, <val hdr 1> ...], <key 2>: ...}
(dictionary).
"""
T = TRACE()
import astropy.io.fits as pyfits
keyDic = defaultdict(list)
try:
for key in keyList:
vals = pyfits.getval(fitsFile, key)
if isinstance(vals, basestring):
vals = [vals]
keyDic[key] = list(vals)
return keyDic
except Exception, e:
msg = ". Error: %s" % str(e)
errMsg = genLog("NGAMS_ER_RETRIEVE_KEYS",
[str(keyList), fitsFile + msg])
logger.exception(errMsg)
raise
def ngamsExampleCacheCtrlPI(srvObj, cacheEntryObj):
"""
Example Cache Control Plug-in to control the cache holding based on
an expiration time for each cached file.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
cacheEntryObj: Cache Entry Object containing the information for the
cached file (ngamsCacheEntry).
Returns: Returns True if the file can be deleted from the cache,
otherwise False (boolean).
"""
T = TRACE()
try:
global ngamsExampleCacheCtrlPI_maxCacheTime
if (not ngamsExampleCacheCtrlPI_maxCacheTime):
plugInPars = srvObj.getCfg().\
getVal("Caching[1].CacheControlPlugInPars")
plugInParDic = ngamsPlugInApi.parseRawPlugInPars(plugInPars)
try:
_maxCacheTime = float(plugInParDic["max_cache_time"])
except:
msg = "Missing plug-in parameter: max_cache_time"
raise Exception, msg
# Check if the cache time is exceeded.
if ((time.time() - cacheEntryObj.getCacheTime()) > _maxCacheTime):
return True
return False
except Exception:
raise
def checkForDblExt(complFilename,
relFilename):
"""
If if the File ID is derived from the URI, it might be that there is a
double extension due to the way the ngamsPlugInApi.genFileInfo() generates
the filename. This function checks double extensions and remove one of them
in case there are two.
complFilename: Complete filename (string).
relFilename: Relative filename (string).
Returns: Tuple with complete filename and relative filename
(tuple/string).
"""
T = TRACE()
filename2, ext1 = os.path.splitext(complFilename)
filename3, ext2 = os.path.splitext(filename2)
if ((ext1 != "") and (ext1 == ext2)):
# Remove one of the extensions.
complFilename = complFilename[0:-len(ext1)]
relFilename = relFilename[0:-len(ext1)]
return (complFilename, relFilename)
def wakeUpHost(srvObj, suspHost):
"""
Wake up a host which has been suspended. After invoking the appropriate
Wake-Up Plug-In it is checked within the time-out defined in the
NG/AMS Configuration File if the server, which was woken up is up
and running. If this is not the case within the specified time-out,
an exception is thrown.
srvObj: Reference to instance of the NG/AMS server (ngamsServer).
suspHost: Host name of the suspended host (string).
Returns: Void.
"""
T = TRACE()
wakeUpPi = srvObj.getCfg().getWakeUpPlugIn()
portNo = srvObj.getDb().getPortNoFromHostId(suspHost)
try:
plugInMethod = loadPlugInEntryPoint(wakeUpPi)
plugInMethod(srvObj, suspHost)
ipAddress = srvObj.getDb().getIpFromHostId(suspHost)
ngamsHighLevelLib.pingServer(ipAddress, portNo,
srvObj.getCfg().getWakeUpCallTimeOut())
except Exception:
logger.exception("Error waking up host %s", suspHost)
raise
def _genObjectStatus():
"""
Generate a report with information about objects allocated, numbers of
references to each object and other information that can be used to
track down memory (object) leaks in the server.
Returns: Object report (string).
"""
T = TRACE()
rep = "NG/AMS SERVER OBJECT STATUS REPORT\n\n"
import gc
gc.set_debug(gc.DEBUG_COLLECTABLE | gc.DEBUG_UNCOLLECTABLE
| gc.DEBUG_INSTANCES | gc.DEBUG_OBJECTS)
rep += "=Garbage Collector Status:\n\n"
rep += "Enabled: %d\n" % gc.isenabled()
rep += "Unreachable objects: %d\n" % gc.collect()
rep += "Threshold: %s\n" % str(gc.get_threshold())
#rep += "Objects:\n"
#rep += str(gc.get_objects()) + "\n"
rep += "Garbage:\n"
rep += str(gc.garbage) + "\n\n"
# Dump refence count status of all objects allocated into file specified.
rep += "=Object Reference Counts:\n\n"
for objInfo in _genRefCountRep():
rep += "%-4d %s\n" % (objInfo[0], objInfo[1])
rep += "\n=EOF"
return rep
def checkStagingAreas(srvObj):
"""
Check the Staging Areas of the disks registered, and in case files
are found on these move htese to the Back-Log Area.
srvObj: Reference to NG/AMS server class object (ngamsServer).
Returns: Void.
"""
T = TRACE()
diskList = ngamsDiskUtils.\
getDiskInfoForMountedDisks(srvObj.getDb(), srvObj.getHostId(),
srvObj.getCfg().\
getRootDirectory())
# Generate first a dictionary with all files in the staging areas.
stagingFileDic = {}
for disk in diskList:
stagingArea = disk.getStagingArea()
if (stagingArea != ""):
fileList = glob.glob(stagingArea + "/*")
fileList += glob.glob(stagingArea + "/.NGAMS*")
for filename in fileList:
stagingFileDic[filename] = 1
# Go through all files in the staging file dictionary and move them to
# the Bad Files Area.
for filename in stagingFileDic.keys():
ngamsHighLevelLib.moveFile2BadDir(srvObj.getCfg(), filename)
def remDisk(srvObj, reqPropsObj, diskId, execute):
"""
Select a disk for removal and remove the information if so specified
from the NGAS DB and delete all information on the disk.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
diskId: ID of disk. Complete ID must be specified. I.e., no
wildcards are handled (string).
execute: If set to 1 the information about the disk will be deleted.
Otherwise only the information about the disk selected for
deletion will be queried (integer/0|1).
Returns: Status object contained information about disk
selected for deletion/deleted (ngamsStatus).
"""
T = TRACE()
tmpFilePat = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(),
"REMDISK_CMD")
try:
status = _remDisk(srvObj, reqPropsObj, diskId, execute, tmpFilePat)
return status
finally:
rmFile(tmpFilePat + "*")
def saveInStagingFile(ngamsCfgObj, reqPropsObj, httpRef, stagingFilename,
diskInfoObj):
"""
Save the data ready on the HTTP channel, into the given Staging
Area file.
ngamsCfgObj: NG/AMS Configuration (ngamsConfig).
reqPropsObj: NG/AMS Request Properties object (ngamsReqProps).
stagingFilename: Staging Area Filename as generated by
ngamsHighLevelLib.genStagingFilename() (string).
diskInfoObj: Disk info object. Only needed if mutual exclusion
is required for disk access (ngamsDiskInfo).
Returns: Void.
"""
T = TRACE()
try:
blockSize = ngamsCfgObj.getBlockSize()
return saveFromHttpToFile(ngamsCfgObj, reqPropsObj, httpRef,
stagingFilename, blockSize, 1, diskInfoObj)
except Exception as e:
errMsg = genLog("NGAMS_ER_PROB_STAGING_AREA",
[stagingFilename, str(e)])
logger.exception(errMsg)
raise
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle CACHEDEL Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
diskId = None
fileId = None
fileVersion = None
for httpPar in reqPropsObj.getHttpParNames():
if (httpPar == "disk_id"):
diskId = reqPropsObj.getHttpPar("disk_id")
elif (httpPar == "file_id"):
fileId = reqPropsObj.getHttpPar("file_id")
elif (httpPar == "file_version"):
fileVersion = int(reqPropsObj.getHttpPar("file_version"))
else:
pass
if ((not diskId) or (not fileId) or (not fileVersion)):
msg = "Must specify disk_id/file_id/file_version for " +\
"CACHEDEL Command"
raise Exception(msg)
cacheDel(srvObj, reqPropsObj, httpRef, diskId, fileId, fileVersion)
def saveInStagingFile(srvObj, ngamsCfgObj, reqPropsObj, stagingFilename,
startByte):
"""
Save the data ready on the HTTP channel, into the given Staging
Area file.
ngamsCfgObj: NG/AMS Configuration (ngamsConfig).
reqPropsObj: NG/AMS Request Properties object (ngamsReqProps).
stagingFilename: Staging Area Filename as generated by
ngamsHighLevelLib.genStagingFilename() (string).
diskInfoObj: Disk info object. Only needed if mutual exclusion
is required for disk access (ngamsDiskInfo).
Returns: Void.
"""
TRACE()
blockSize = ngamsCfgObj.getBlockSize()
fetchMethod = 'HTTP'
if ngamsCfgObj.getVal("Mirroring[1].fetch_method"):
fetchMethod = ngamsCfgObj.getVal("Mirroring[1].fetch_method")
if fetchMethod == 'RSYNC':
info = ngamsCmd_RSYNCFETCH.saveToFile(srvObj, ngamsCfgObj, reqPropsObj,
stagingFilename, blockSize,
startByte)
else:
info = ngamsCmd_HTTPFETCH.saveToFile(srvObj, ngamsCfgObj, reqPropsObj,
stagingFilename, blockSize,
startByte)
return info
def _handleFileList(srvObj, reqPropsObj, httpRef):
"""
Handle STATUS?file_list... Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: The File List ID allocated for this request (string).
"""
T = TRACE()
# Should a lower limit for the ingestion date be taken into account.
fromIngDate = None
if (reqPropsObj.hasHttpPar("from_ingestion_date")):
tmpTromIngDate = reqPropsObj.getHttpPar("from_ingestion_date")
fromIngDate = fromiso8601(tmpTromIngDate)
# Handle the unique flag. If this is specified, only information for unique
# File ID/Version pairs are returned.
unique = False
if (reqPropsObj.hasHttpPar("unique")):
unique = int(reqPropsObj.getHttpPar("unique"))
# Dump the file information needed.
fileListId = genUniqueId()
dbmBaseName = STATUS_FILE_LIST_DBM_TAG % fileListId
fileInfoDbmName = ngamsHighLevelLib.genTmpFilename(srvObj.getCfg(),
dbmBaseName)
# Dump the file info from the DB. Deal with uniqueness quickly
# (instead of using a new file like before)
try:
fileInfoDbm = ngamsDbm.ngamsDbm(fileInfoDbmName, 0, 1)
fileCount = 1
unique_files = set()
for f in srvObj.db.files_in_host(srvObj.getHostId(),
from_date=fromIngDate):
if unique:
key = str('%s_%d' % (f[2], f[3]))
if key in unique_files:
continue
else:
key = str(fileCount)
fileInfoDbm.add(key, f)
fileCount += 1
except Exception as e:
rmFile(fileInfoDbmName)
msg = "Problem generating file list for STATUS Command. " +\
"Parameters: from_ingestion_date=%s. Error: %s" %\
(str(fromIngDate), str(e))
raise Exception(msg)
return fileListId
def extractData(result, resourceId, verbose=0):
"""
INPUT:
result: list of strings containing multipart messages
RETURNS:
xyData: dictionary,
"""
T = TRACE(3)
if (verbose >= 3):
print "Entering extractData with parameters " + \
"type(result): %s, resourceId: %s" % (str(type(result)),
resourceId)
eP = MultipartHandler.Parser()
xData = []
yData = []
xyData = {}
if len(result) == 0:
errMsg = "[ERROR]: No data returned. Check whether resourceId is correct: %s" % resourceId
raise Exception(errMsg)
resultClean = []
for res in result:
if len(res) != 0:
resultClean.append(res)
else:
logger.warning("Empty document detected and ignored!")
for res in resultClean:
try:
msg = eP.parsestr(res) # and parse the string
except Exception:
logger.exception("email parsing failed")
raise
# get the first xml part of the email...
xmlParts = MultipartHandler.getMimeParts(msg, 'text/xml')
if len(xmlParts) == 0:
errMsg = "MonitorDataCli.extractData: No text/xml part found!"
raise Exception(errMsg)
else:
xml = xmlParts[0].get_payload() # There should only be one!
# ...and parse it
try:
root = MultipartHandler.VOTable.parseString(xml)
except Exception, e:
# print xml
errMsg = "MonitorDataCli.extractData: XML parsing failed: %s " % str(
e)
raise Exception(errMsg)
try:
(res, cidI,
cids) = MultipartHandler.interpretVotable(root,
selection=resourceId,
verbose=verbose)
except Exception, e:
errMsg = "ERROR interpreting VOTABLE: %s" % str(e)
raise Exception(errMsg)
def locateArchiveFile(srvObj,
fileId,
fileVersion = -1,
diskId = "",
hostId = "",
reqPropsObj = None):
"""
Locate the file indicated by the File ID. Returns a list containing
the necessary information for retrieving the file:
[<Location>, <File Host>, <IP Address>, <Port No>, <Mount Point>,
<Filename>, <File ID>, <File Version>, <Mime-Type>]
- whereby:
<Location> = Location of the file (NGAMS_HOST_LOCAL,
NGAMS_HOST_CLUSTER, NGAMS_HOST_DOMAIN,
NGAMS_HOST_REMOTE).
<File Host> = Host ID of host to be contacted to get access to the
file.
<IP Address> = IP Address of host to be contacted to get access to the
file.
<Port No> = Port number used by the NG/AMS Server.
<Mount Point> = Mount point at which the file is residing.
<Filename> = Name of file relative to mount point.
<File ID> = ID of file.
<File Version> = Version of file.
<Mime-Type> = Mime-type of file (as registered in NGAS).
srvObj: Reference to NG/AMS server class object (ngamsServer).
fileId: File ID of file to locate (string).
fileVersion: Version of the file (integer).
diskId: ID of the disk hosting the file (string).
hostId: ID of the host where the file is located (string).
reqPropsObj: Request Property object to keep track of actions done during
the request handling (ngamsReqProps|None).
Returns: List with information about file location (list).
"""
T = TRACE()
# Get a list with the candidate files matching the query conditions.
res = srvObj.getDb().getFileInfoFromFileId(fileId, fileVersion, diskId,
ignore=0, dbCursor=False)
# r[-2] is the host_id, r[-1] is the mount point
all_info = []
for r in res:
file_info = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(r)
all_info.append((file_info, r[-2], r[-1]))
return _locateArchiveFile(srvObj, fileId, fileVersion, diskId, hostId,
reqPropsObj, all_info)
def ngamsZHLDapi(srvObj, reqPropsObj):
"""
Generic Data Archiving Plug-In to handle archiving of any file.
srvObj: Reference to NG/AMS Server Object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Standard NG/AMS Data Archiving Plug-In Status
as generated by: ngamsPlugInApi.genDapiSuccessStat()
(ngamsDapiStatus).
"""
T = TRACE()
# For now the exception handling is pretty basic:
# If something goes wrong during the handling it is tried to
# move the temporary file to the Bad Files Area of the disk.
logger.debug("Plug-In handling data for file: %s",
os.path.basename(reqPropsObj.getFileUri()))
try:
parDic = {}
handlePars(reqPropsObj, parDic)
diskInfo = reqPropsObj.getTargDiskInfo()
stgFile = reqPropsObj.getStagingFilename()
ext = os.path.splitext(stgFile)[1][1:]
# Generate file information.
logger.debug("Generate file information")
dateDir = toiso8601(fmt=FMT_DATE_ONLY)
fileVersion, relPath, relFilename,\
complFilename, fileExists =\
ngamsPlugInApi.genFileInfo(srvObj.getDb(),
srvObj.getCfg(),
reqPropsObj, diskInfo,
reqPropsObj.\
getStagingFilename(),
parDic[FILE_ID],
parDic[FILE_ID], [dateDir])
complFilename, relFilename = checkForDblExt(complFilename, relFilename)
# Compress the file if requested.
uncomprSize, archFileSize, format, compression, comprExt =\
compressFile(srvObj, reqPropsObj, parDic)
if (comprExt != ""):
complFilename += ".%s" % comprExt
relFilename += ".%s" % comprExt
logger.debug(
"DAPI finished processing file - returning to host application")
insertFitsRecords(srvObj, reqPropsObj, stgFile)
return ngamsPlugInApi.genDapiSuccessStat(
diskInfo.getDiskId(), relFilename, parDic[FILE_ID], fileVersion,
format, archFileSize, uncomprSize, compression, relPath,
diskInfo.getSlotId(), fileExists, complFilename)
except Exception:
logger.exception("Error occurred in DAPI")
raise
def checkSpuriousFiles(srvObj,
tmpFilePat=None,
hostId=None,
diskId=None,
fileId=None,
fileVersion=None):
"""
Check if there are any spurious files in the DB either marked as to be
ignored or having a status indicating a problem. If such are found
according to the criterias defined, these are added in a DBM DB, which
name is returned.
srvObj: Reference to NG/AMS server class object (ngamsServer).
tmpFilePat: Pattern to apply for temporary files (string).
hostId: Name of NGAS host on which the files reside (string).
diskId: Disk ID of disk to take into account (string|None).
fileId: File ID of file(s) to take into account (string|None).
fileVersion: Version of file(s) to take into account (integer|None).
Returns: Returns name of DBM DB with references to spurious
files found (string).
"""
T = TRACE()
if (hostId == ""): hostId = None
if (diskId == ""): diskId = None
if (fileId == ""): fileId = None
if (fileVersion == -1): fileVersion = None
# DBM DB containing information about spurious files.
filename = "_SPURIOUS_FILES"
if (tmpFilePat):
spuriousFilesDbmName = os.path.normpath(tmpFilePat + filename)
else:
spuriousFilesDbmName = ngamsHighLevelLib.\
genTmpFilename(srvObj.getCfg(), filename)
spuriousFilesDbm = ngamsDbm.ngamsDbm(spuriousFilesDbmName, writePerm=1)
# Check that there are no spurious files in connection with this disk in
# the DB (where ngas_files.file_ignore != 0 or ngas_files.status != "1*******"
files = srvObj.db.getFileSummarySpuriousFiles1(hostId,
diskId,
fileId,
fileVersion,
fetch_size=200)
for fileInfo in files:
spuriousFilesDbm.addIncKey(fileInfo)
spuriousFilesDbm.sync()
del spuriousFilesDbm
return spuriousFilesDbmName
def _checkFileAccess(srvObj,
reqPropsObj,
httpRef,
fileId,
fileVersion=-1,
diskId=""):
"""
Check if a file is accessible either on the local host or on a remotely
located host.
srvObj: Instance of the server object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
fileId: File ID (string).
fileVersion: File Version (integer).
diskId: Disk ID (string).
Returns: Returns message indicating if the file is available
(string).
"""
T = TRACE()
logger.debug("Checking for access to file with ID: %s", fileId)
# Check if the file is located on this host, or if the request should be
# forwarded (if this server should act as proxy).
location, fileHost, ipAddress, filePortNo, mountPoint, filename, fileId,\
fileVersion, mimeType =\
ngamsFileUtils.locateArchiveFile(srvObj, fileId, fileVersion,
diskId)
if (location != NGAMS_HOST_LOCAL):
# Go and get it!
host, port = srvObj.get_remote_server_endpoint(fileHost)
pars = (('file_access', fileId), ('file_version', fileVersion),
('disk_id', diskId))
resp = ngamsHttpUtils.httpGet(host, port, 'STATUS', pars, timeout=60)
with contextlib.closing(resp):
return ngamsStatus.to_status(resp, fileHost, 'STATUS').getMessage()
else:
# First check if this system allows for Retrieve Requests.
if (not srvObj.getCfg().getAllowRetrieveReq()):
msg = genLog("NGAMS_INFO_SERVICE_UNAVAIL", ["File Retrieval"])
else:
fileHost = "%s:%d" % (getHostName(), filePortNo)
msg = genLog("NGAMS_INFO_FILE_AVAIL",
[fileId + "/Version: " + str(fileVersion), fileHost])
return msg
def handlePars(reqPropsObj, parDic):
"""
Parse/handle the HTTP parameters.
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
parDic: Dictionary with the parameters (dictionary).
Returns: Void.
"""
T = TRACE()
# Get parameters.
logger.debug("Get request parameters")
parDic[TARG_MIME_TYPE] = None
parDic[FILE_ID] = None
parDic[VERSIONING] = 1
parDic[COMPRESSION] = None
parDic[COMPRESSION_EXT] = None
if (reqPropsObj.hasHttpPar(TARG_MIME_TYPE)):
parDic[TARG_MIME_TYPE] = reqPropsObj.getHttpPar(TARG_MIME_TYPE)
# If the file_id is not given, we derive it from the name of the URI.
if (reqPropsObj.hasHttpPar(FILE_ID)):
parDic[FILE_ID] = reqPropsObj.getHttpPar(FILE_ID)
if (not parDic[FILE_ID]):
if (reqPropsObj.getFileUri().find("file_id=") > 0):
file_id = reqPropsObj.getFileUri().split("file_id=")[1]
parDic[FILE_ID] = file_id
logger.info("No file_id given, but found one in the URI: %s",
parDic[FILE_ID])
else:
parDic[FILE_ID] = os.path.basename(reqPropsObj.getFileUri())
logger.info("No file_id given, using basename of URI: %s",
parDic[FILE_ID])
if (reqPropsObj.hasHttpPar(VERSIONING)):
parDic[VERSIONING] = int(reqPropsObj.getHttpPar(VERSIONING))
# Set also the no_versioning parameter for backwards compatibility reasons
if (parDic[VERSIONING]):
reqPropsObj.addHttpPar("no_versioning", "0")
else:
reqPropsObj.addHttpPar("no_versioning", "1")
if (reqPropsObj.hasHttpPar(COMPRESSION)):
parDic[COMPRESSION] = reqPropsObj.getHttpPar(COMPRESSION)
if (reqPropsObj.hasHttpPar(COMPRESSION_EXT)):
parDic[COMPRESSION_EXT] = reqPropsObj.getHttpPar(COMPRESSION_EXT)
if ((parDic[COMPRESSION] and (parDic[COMPRESSION_EXT] == None)) or
(not parDic[COMPRESSION] and (parDic[COMPRESSION_EXT] != None))):
raise Exception(
genLog("NGAMS_ER_DAPI", [
"Parameters compression and compression_ext"
"must be given together."
]))
def ngamsSuspensionPlugIn(srvObj):
"""
Suspension Plug-In to suspend an NGAS host.
srvObj: Reference to instance of the NG/AMS Server (ngamsServer).
Returns: Void.
"""
T = TRACE()
commands.getstatusoutput("sudo /sbin/shutdown -h now")
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Handle the RUN TASK (RUNTASK) Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
global queScanThread
httpMethod = reqPropsObj.getHttpMethod()
if (httpMethod != 'POST'):
errMsg = 'OK'
if (reqPropsObj.hasHttpPar('action')):
action_req = reqPropsObj.getHttpPar('action')
if ('cancel' == action_req):
if (reqPropsObj.hasHttpPar('task_id')):
taskId = reqPropsObj.getHttpPar('task_id')
cancelDict[taskId] = 1
if (g_mrLocalTask and taskId == g_mrLocalTask._taskId):
res = g_mrLocalTask.stop()
if (res[0]):
errMsg = 'Cancel result failed: %s\n' % str(res[1])
else:
errMsg = 'task_id is missing'
else:
errMsg = 'Unknown RUNTASK command action %s' % action_req
else:
errMsg = 'RUNTASK command needs action for GET request\n'
httpRef.send_data(errMsg, NGAMS_TEXT_MT)
else:
postContent = _getPostContent(srvObj, reqPropsObj, httpRef)
mrLocalTask = pickle.loads(postContent)
if (not mrLocalTask):
errMsg = 'Cannot instantiate local task from POST'
mrr = MRLocalTaskResult(None, -2, errMsg)
httpRef.send_data(pickle.dumps(mrr), NGAMS_TEXT_MT)
else:
logger.debug('Local task %s is submitted', mrLocalTask._taskId)
mrr = MRLocalTaskResult(mrLocalTask._taskId, 0, '')
httpRef.send_data(pickle.dumps(mrr), NGAMS_TEXT_MT)
args = (srvObj, mrLocalTask)
scheduleThread = threading.Thread(None, _scheduleQScanThread,
'SCHEDULE_THRD', args)
scheduleThread.setDaemon(0)
scheduleThread.start()
def ngamsGenericOfflinePlugIn(srvObj, reqPropsObj=None):
"""
Generic NGAS Offline Plug-In.
srvObj: Reference to instance of the NG/AMS Server class
(ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: Void.
"""
T = TRACE()
def ngamsTestWakeUpPlugIn(srvObj, hostId):
"""
Dummy Wake-Up Plug-In to test the handling of the NGAS host suspension.
srvObj: Reference to instance of the NG/AMS Server (ngamsServer).
hostId: Name of NGAS host to be woken up (string).
Returns: Void.
"""
T = TRACE(2)
srvObj.getDb().resetWakeUpCall(hostId, 1)
def __handleCmd(srvObj, reqPropsObj):
"""
Handle the Mirroring Archive (MIRRARCHIVE) Command.
srvObj: Reference to NG/AMS server class object (ngamsServer).setState
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
Returns: Void.
"""
TRACE()
# Is this NG/AMS permitted to handle Archive Requests?
logger.debug(
"Checking if this NG/AMS permitted to handle Archive Requests?")
if (not srvObj.getCfg().getAllowArchiveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Archive"])
raise Exception, errMsg
# Generate staging filename.
stgFilename = reqPropsObj.getStagingFilename()
logger.info("staging filename is: %s", stgFilename)
startByte = 0
if (os.path.exists(stgFilename) == 0):
logger.debug('this is a new staging file')
else:
startByte = os.path.getsize(stgFilename)
logger.debug(
'staging file already exists, requesting resumption of download from byte %d',
startByte)
# Set reference in request handle object to the read socket.
try:
# Retrieve file_id and file_version from request proposal
file_id = reqPropsObj.fileinfo['fileId']
file_version = reqPropsObj.fileinfo['fileVersion']
logger.debug("Got file_id=%s and file_version=%s", file_id,
file_version)
# Retrieve file contents (from URL, archive pull, or by storing the body
# of the HTTP request, archive push).
logger.info("Saving in staging file: %s", stgFilename)
stagingInfo = saveInStagingFile(srvObj, srvObj.getCfg(), reqPropsObj,
stgFilename, startByte)
reqPropsObj.incIoTime(stagingInfo[0])
checksumPlugIn = "ngamsGenCrc32"
checksum = stagingInfo[1]
except ngamsFailedDownloadException.FailedDownloadException, e:
raise
def handlePars(reqPropsObj,
parDic):
"""
Parse/handle the HTTP parameters.
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
parDic: Dictionary with the parameters (dictionary).
Returns: Void.
"""
T = TRACE()
# Get parameters.
logger.debug("Get request parameters")
parDic[TARG_MIME_TYPE] = None
parDic[FILE_ID] = None
parDic[VERSIONING] = 1
if (reqPropsObj.hasHttpPar(TARG_MIME_TYPE)):
parDic[TARG_MIME_TYPE] = reqPropsObj.getHttpPar(TARG_MIME_TYPE)
# If the file_id is not given, we derive it from the name of the URI.
if (reqPropsObj.hasHttpPar(FILE_ID)):
parDic[FILE_ID] = reqPropsObj.getHttpPar(FILE_ID)
if (not parDic[FILE_ID]):
if (reqPropsObj.getFileUri().find("file_id=") > 0):
file_id = reqPropsObj.getFileUri().split("file_id=")[1]
parDic[FILE_ID] = os.path.basename(file_id)
logger.info("No file_id given, but found one in the URI: %s", parDic[FILE_ID])
else:
parDic[FILE_ID] = os.path.basename(reqPropsObj.getFileUri())
logger.info("No file_id given, using basename of fileUri: %s",
parDic[FILE_ID])
# TODO: It would seem like 'versioning' is not used anywhere,
# but still gets more priority than 'no_versioning'.
# which is used everywhere
if VERSIONING in reqPropsObj:
parDic[VERSIONING] = int(reqPropsObj[VERSIONING])
elif 'no_versioning' in reqPropsObj:
parDic[VERSIONING] = 0 if int(reqPropsObj['no_versioning']) else 1
# Set also the no_versioning parameter for backwards compatibility reasons
if (parDic[VERSIONING]):
reqPropsObj.addHttpPar("no_versioning", "0")
else:
reqPropsObj.addHttpPar("no_versioning", "1")
extract_compression_params(reqPropsObj, parDic)
def ngamsDiskSyncPlugIn(srvObj):
"""
Disk Sync Plug-In to flush the cache of the 3ware controller.
srvObj: Reference to instance of NG/AMS Server class (ngamsServer).
Returns: Void.
"""
T = TRACE()
# Sync filesystem to ensure file received on disk.
logger.debug("Performing OS sync command ...")
with _proc_startup_lock:
subprocess.call("sync")
def rescanCont(contId):
"""
Rescan a 3ware controller.
contId: ID (number) of the controller (integer).
Returns: Void.
"""
return # Don't want to execute this
T = TRACE()
cmd = "sudo /usr/local/sbin/tw_cli /c%d rescan" % (int(contId))
logger.debug("Invoking command to rescan 3ware unit: %s ...", cmd)
stat, out = commands.getstatusoutput(cmd)
logger.debug("Result of command: %s to rescan 3ware unit: %d", cmd, stat)
def handleCmd(srvObj,
reqPropsObj,
httpRef):
"""
Handle CLONE command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
# Is this NG/AMS permitted to handle Archive Requests?
if (not srvObj.getCfg().getAllowArchiveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Clone"])
raise Exception(errMsg)
# Check if State/Sub-State correct for perfoming the cloning.
srvObj.checkSetState("Command CLONE", [NGAMS_ONLINE_STATE],
[NGAMS_IDLE_SUBSTATE, NGAMS_BUSY_SUBSTATE])
# Get the parameters from the query.
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
else:
fileId =""
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
else:
diskId = ""
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = int(reqPropsObj.getHttpPar("file_version"))
else:
fileVersion = -1
if (reqPropsObj.hasHttpPar("target_disk_id")):
targetDiskId = reqPropsObj.getHttpPar("target_disk_id")
else:
targetDiskId = ""
# Carry out the cloning.
clone(srvObj, diskId, fileId, fileVersion,targetDiskId,reqPropsObj,httpRef)
def addSubscriber(srvObj, subscrObj):
"""
Add a Subscriber to the list of Subscribers. The information about
the Subscriber is also updated in the DB.
srvObj: Reference to NG/AMS Server object (ngamsServer).
subscrObj: Subscriber Object (ngamsSubscriber).
Returns: Void.
"""
T = TRACE()
srvObj.getDb().insertSubscriberEntry(subscrObj)
#subscrObj.write(srvObj.getDb())
srvObj.registerSubscriber(subscrObj)