def handleCmd(srvObj,
reqPropsObj,
httpRef):
"""
Handle STATUS command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler object
(ngamsHttpRequestHandler).
Returns: Void.
"""
status = ngamsStatus.ngamsStatus()
status.setDate(toiso8601()).\
setVersion(getNgamsVersion()).setHostId(srvObj.getHostId()).\
setStatus(NGAMS_SUCCESS).\
setMessage("Successfully handled command STATUS").\
setState(srvObj.getState()).setSubState(srvObj.getSubState())
reqPropsObjRef = reqPropsObj
# Get the information requested.
diskId = ""
fileId = ""
fileVersion = ""
configurationFile = ""
fileAccess = ""
hostId = ""
requestId = ""
dbTime = ""
dbTimeReset = ""
fileList = ""
fileListId = ""
maxElements = 100000
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = int(reqPropsObj.getHttpPar("file_version"))
if (reqPropsObj.hasHttpPar("configuration_file")):
configurationFile = reqPropsObj.getHttpPar("configuration_file")
if (reqPropsObj.hasHttpPar("file_access")):
fileAccess = reqPropsObj.getHttpPar("file_access")
if (reqPropsObj.hasHttpPar("host_id")):
hostId = reqPropsObj.getHttpPar("host_id")
if (reqPropsObj.hasHttpPar("max_elements")):
maxElements = int(reqPropsObj.getHttpPar("max_elements"))
if (reqPropsObj.hasHttpPar("request_id")):
requestId = reqPropsObj.getHttpPar("request_id")
if (reqPropsObj.hasHttpPar("db_time")):
dbTime = True
if (reqPropsObj.hasHttpPar("db_time_reset")):
dbTimeReset = True
if (reqPropsObj.hasHttpPar("flush_log")):
# in the past this called flushLog()
# do we really want to keep that functionality? I doubt it
pass
if (reqPropsObj.hasHttpPar("file_list")):
fileList = int(reqPropsObj.getHttpPar("file_list"))
if (reqPropsObj.hasHttpPar("file_list_id")):
fileListId = reqPropsObj.getHttpPar("file_list_id")
if (reqPropsObj.hasHttpPar("dump_object_info")):
# Dump status of all objects allocated into file specified.
targFile = reqPropsObj.getHttpPar("dump_object_info")
rmFile(targFile)
fo = open(targFile, "w")
fo.write(_genObjectStatus())
fo.close()
# Handle request for file info.
genCfgStatus = 0
genDiskStatus = 0
genFileStatus = 0
genStatesStatus = 1
genRequestStatus = 0
msg = ""
help = 0
if (reqPropsObj.hasHttpPar("help")):
global _help
msg = _help
help = 1
elif hostId and hostId != srvObj.getHostId():
host, port = srvObj.get_remote_server_endpoint(hostId)
cfgObj = srvObj.getCfg()
if not cfgObj.getProxyMode():
httpRef.redirect(host, port)
return
else:
try:
httpRef.proxy_request(hostId, host, port)
except Exception as e:
ex = re.sub("<|>", "", str(e))
errMsg = genLog("NGAMS_ER_COM",
[host, port,ex])
raise Exception(errMsg)
return
elif (fileList):
if (not fileListId):
# It's a new STATUS?file_list request.
fileListId = _handleFileList(srvObj, reqPropsObj, httpRef)
# Send back data from the request.
_handleFileListReply(srvObj, reqPropsObj, httpRef, fileListId,
maxElements)
elif (diskId):
diskObj = ngamsDiskInfo.ngamsDiskInfo()
diskObj.read(srvObj.getDb(), diskId)
status.addDiskStatus(diskObj)
genDiskStatus = 1
elif (fileId):
if (not fileVersion): fileVersion = -1
try:
fileObj = ngamsFileInfo.ngamsFileInfo()
fileObj.read(srvObj.getHostId(), srvObj.getDb(), fileId, fileVersion)
diskObj = ngamsDiskInfo.ngamsDiskInfo()
try:
diskObj.read(srvObj.getDb(), fileObj.getDiskId())
except:
errMsg = "Illegal Disk ID found: {0} for file with ID: {1}".format(fileObj.getDiskId(), fileId)
raise Exception(errMsg)
diskObj.addFileObj(fileObj)
status.addDiskStatus(diskObj)
except:
# The file was not found in the database. Check if it is available
# on a remote partner site.
host, port, status_info, disk_info, file_info = \
ngamsFileUtils.lookup_partner_site_file_status(srvObj, fileId,
fileVersion,
reqPropsObj)
# Update status reply using the disk status and file status
# information retrieved from the partner site
status.addDiskStatus(disk_info)
genDiskStatus = 1
genFileStatus = 1
elif (requestId):
logger.debug("Checking status of request with ID: %s", requestId)
reqPropsObjRef = srvObj.getRequest(requestId)
if (not reqPropsObjRef):
errMsg = genLog("NGAMS_ER_ILL_REQ_ID", [requestId])
raise Exception(errMsg)
genRequestStatus = 1
elif (configurationFile):
msg = "configuration_file=" + srvObj.cfg_fname
genCfgStatus = 1
# Hidden feature to return complete Cfg!!
try:
genCfgStatus = int(configurationFile)
except ValueError:
genCfgStatus = 1
status.setNgamsCfgObj(srvObj.getCfg())
elif (fileAccess):
if (not fileVersion): fileVersion = -1
fileId = fileAccess
msg = _checkFileAccess(srvObj, reqPropsObj, httpRef, fileId,
fileVersion, diskId)
elif (dbTime):
logger.debug("Querying total DB time")
msg = "Total DB time: %.6fs" % srvObj.getDb().getDbTime()
elif (dbTimeReset):
msg = "Resetting DB timer"
logger.debug(msg)
srvObj.getDb().resetDbTime()
else:
msg = "Successfully handled command STATUS"
if (reqPropsObjRef == reqPropsObj):
reqPropsObj.setCompletionTime()
srvObj.updateRequestDb(reqPropsObj)
if (genCfgStatus or genDiskStatus or genFileStatus or genRequestStatus):
status.setReqStatFromReqPropsObj(reqPropsObjRef)
# Generate XML reply.
logger.debug("Status requests: " + str([genCfgStatus, genDiskStatus, genFileStatus,
genStatesStatus]))
xmlStat = status.genXmlDoc(genCfgStatus, genDiskStatus, genFileStatus,
genStatesStatus)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT)
elif not httpRef.reply_sent:
httpRef.send_status(msg)
if (msg and (not help)):
logger.info(msg)
else:
logger.info("Successfully handled command STATUS")
def runAllTests(notifEmail=None, skip=None):
"""
Run all tests in ngasUtils/test.
notifEmail: List of email recipients that should be notified about the
test results (list/email addresses).
skip: Test Suites or Test Cases to skip. Comma separated list
(string).
Returns: Void.
"""
skipDic = {}
if (skip):
for test in skip.split(","):
skipDic[test] = 1
testModList = getTestList()
startTime = time.time()
failModDic = {}
noOfTests = len(testModList)
testCount = 0
format = "Running Test Suite: %-32s %-8s"
line = "\nNGAS UTILITIES FUNCTIONAL TESTS - TEST REPORT\n"
print line
testRep = line + "\n"
line = "Date: %s" % PccUtTime.TimeStamp().getTimeStamp()
print line
testRep += line + "\n"
line = "Host: %s" % getHostName()
print line
testRep += line + "\n"
line = "NG/AMS Version: %s\n" % getNgamsVersion()
print line
testRep += line + "\n"
for mod in testModList:
if (skipDic.has_key(mod)): continue
testCount += 1
line = format % (mod, ("(#%d/%d)" % (testCount, noOfTests)))
testRep += line
sys.stdout.write(line)
sys.stdout.flush()
suiteStartTime = time.time()
stat, stdout, stderr = ngamsCore.execCmd("python " + mod + ".py",
NGAMS_TEST_MAX_TS_TIME)
testTime = (time.time() - suiteStartTime)
if (testTime >= NGAMS_TEST_MAX_TS_TIME):
failModDic[mod] = "TIME-OUT"
stat = " - %-5.1fs - TIME-OUT!!\n" % testTime
elif (stdout.find("FAILED") != -1):
failModDic[mod] = stdout + " --- " + stderr
stat = " - %-5.1fs - FAILURE!!\n" % testTime
else:
stat = " - %-5.1fs - SUCCESS\n" % testTime
sys.stdout.write(stat)
sys.stdout.flush()
testRep += stat
execTime = (time.time() - startTime)
line = "\n\nExecuted %d Test Suites in %.3f seconds\n\n" %\
(len(testModList), execTime)
print line
testRep += line
if (failModDic):
line = 80 * "="
print line
testRep += line
line = "\n\nFAILED TEST SUITES:\n\n"
print line
testRep += line
failMods = failModDic.keys()
failMods.sort()
for failMod in failMods:
line = "\n%s:\n%s\n" % (failMod, failModDic[failMod])
print line
testRep += line + "\n"
line = 80 * "-"
print line
testRep += line
line = 80 * "="
print line
testRep += line
# Send out email with test report if requested.
if (notifEmail):
notifEmail = notifEmail.split(",")
par = "NgamsCfg.Server[1].MountRootDirectory"
ngamsCfgObj = ngamsConfig.ngamsConfig().storeVal(par, "/NGAS")
ngamsHighLevelLib.sendEmail(ngamsCfgObj, "localhost",
"NGAS UTILITIES FUNCTIONAL TESTS REPORT",
notifEmail, "ngas@%s" %\
ngamsLib.getCompleteHostName(), testRep)
def main():
"""
Entry point for the ngamsPClient script
"""
DUMP_TO_STDOUT = object()
parser = argparse.ArgumentParser(usage="ngamsPClient [-h] <options> cmd")
parser.add_argument('cmd', nargs='?', help='Command to issue')
gparser = parser.add_argument_group('General options')
gparser.add_argument('-L',
'--license',
help='Show the license information',
action='store_true')
gparser.add_argument('-V',
'--version',
help='Show the version and exit',
action='store_true')
gparser.add_argument('-v',
'--verbose',
help='Increase verbosity. More -v is more verbose',
action='count',
default=3)
gparser.add_argument(
'-s',
'--status',
metavar="FILE",
nargs='?',
const=DUMP_TO_STDOUT,
help=
'Display the XML status document of the command. If FILE is given the document written to disk',
)
cparser = parser.add_argument_group('Connection options')
cparser.add_argument('-H',
'--host',
help='Host to connect to',
default="127.0.0.1")
cparser.add_argument('-p',
'--port',
help='Port to connect to',
default=7777,
type=int)
cparser.add_argument('-t',
'--timeout',
help='Connection timeout, in sec.',
default=None,
type=float)
cparser.add_argument(
'--servers', help='A comma-separated list of host:server addresses')
parser.add_argument(
'-P',
'--param',
help=
'Additional HTTP parameters in the form of param=value, can be specified more than once',
action='append',
default=[])
parser.add_argument('-r',
'--reload',
help='Reload the module implementing the command',
action='store_true')
parser.add_argument('-m',
'--mime-type',
help='The mime-type',
default='application/octet-stream')
parser.add_argument(
'-o',
'--output',
help='File/directory where to store the retrieved data')
parser.add_argument('-a',
'--async',
help='Run command asynchronously',
action='store_true',
dest='asynchronous')
parser.add_argument('-A',
'--auth',
help='BASIC authorization string (user:password)')
parser.add_argument('-F',
'--force',
help='Force the action',
action='store_true')
parser.add_argument('-f', '--file-id', help='Indicates a File ID')
parser.add_argument('--file-version',
help='A file version',
type=int,
default=-1)
parser.add_argument('--file-id-list', help='A list of File IDs')
parser.add_argument('--file-uri', help='A File URI')
parser.add_argument('--file-info-xml', help='An XML File Info string')
parser.add_argument('-n',
'--no-versioning',
help='Do not increase the file version',
action='store_true')
parser.add_argument('-d', '--disk-id', help='Indicates a Disk ID')
parser.add_argument('-e',
'--execute',
help='Executes the action',
action='store_true')
parser.add_argument('--path', help='File path')
parser.add_argument('--slot-id', help='The Slot ID for the label')
parser.add_argument('--host-id', help='The Host ID')
parser.add_argument(
'--p-plugin',
help='Processing plug-in to apply before retrieving data')
parser.add_argument(
'--p-plugin-pars',
help=
'Parameters for the processing plug-in, can be specified more than once',
action='append')
sparser = parser.add_argument_group('Subscription options')
sparser.add_argument('-u', '--url', help='URL to subscribe/unsubscribe')
sparser.add_argument('--priority',
help='Priority used for subscription',
type=int,
default=None)
sparser.add_argument('--f-plugin',
help='Filtering plug-in to use for this subscription')
sparser.add_argument(
'--f-plugin-pars',
help=
'Parameters for the filtering plug-in, can be specified more than once',
action='append')
sparser.add_argument('-S',
'--start-date',
help='Start date for subscription')
cparser = parser.add_argument_group('Container options')
cparser.add_argument('-c',
'--container-id',
help='Specifies a Container ID')
cparser.add_argument('--container-name', help='Specifies a Container name')
cparser.add_argument(
'--container-hierarchy',
help='Specifies a Container Hierarchy as an XML document')
cparser.add_argument('--parent-container-id',
help='Specifies a Parent Container ID')
cparser.add_argument('-C',
'--close',
help='Closes the container after appending a file',
action='store_true')
cparser.add_argument('--recursive',
help='Recursively remove containers',
action='store_true')
opts = parser.parse_args()
setup_logging(opts)
if opts.version:
print(getNgamsVersion())
return
elif opts.license:
print(getNgamsLicense())
return
elif not opts.cmd:
parser.error('cmd required but none given')
if opts.servers:
servers = [
(parts[0], int(parts[1]))
for parts in (server.split(':')
for server in filter(None, opts.servers.split(',')))
]
client = ngamsPClient(servers=servers,
timeout=opts.timeout,
auth=opts.auth,
reload_mod=opts.reload)
else:
client = ngamsPClient(opts.host,
opts.port,
timeout=opts.timeout,
auth=opts.auth,
reload_mod=opts.reload)
# Invoke the proper operation.
cmd = opts.cmd
mtype = opts.mime_type
pars = [p.split('=') for p in opts.param]
if cmd in [NGAMS_ARCHIVE_CMD, 'QARCHIVE']:
if not opts.file_uri:
msg = "Must specify parameter --file-uri for a ARCHIVE/QARCHIVE commands"
raise Exception(msg)
pars += [('file_version',
opts.file_version)] if opts.file_version is not None else []
stat = client.archive(opts.file_uri,
mtype,
opts.asynchronous,
opts.no_versioning,
cmd=cmd,
pars=pars)
elif cmd == "CARCHIVE":
stat = client.carchive(opts.file_uri, mtype)
elif cmd == "CAPPEND":
stat = client.cappend(opts.file_id, opts.file_id_list,
opts.container_id, opts.container_name,
opts.force, opts.close)
elif cmd == "CCREATE":
stat = client.ccreate(opts.container_name, opts.parent_container_id,
opts.container_hierarchy)
elif cmd == "CDESTROY":
stat = client.cdestroy(opts.container_name, opts.container_id,
opts.recursive)
elif cmd == "CLIST":
stat = client.clist(opts.container_name, opts.container_id)
elif cmd == "CREMOVE":
stat = client.cremove(opts.file_id, opts.file_id_list,
opts.container_id, opts.container_name)
elif cmd == 'CRETRIEVE':
stat = client.cretrieve(opts.container_name, opts.container_id,
opts.output)
elif (cmd == NGAMS_CACHEDEL_CMD):
pars.append(("disk_id", opts.disk_id))
pars.append(("file_id", opts.file_id))
pars.append(("file_version", str(opts.file_version)))
stat = client.get_status(cmd, pars=pars)
elif (cmd == NGAMS_CLONE_CMD):
stat = client.clone(opts.file_id, opts.disk_id, opts.file_version,
opts.asynchronous)
elif (cmd == NGAMS_EXIT_CMD):
stat = client.exit()
elif (cmd == NGAMS_INIT_CMD):
stat = client.init()
elif (cmd == NGAMS_LABEL_CMD):
stat = client.label(opts.slot_id, opts.host_id)
elif (cmd == NGAMS_OFFLINE_CMD):
stat = client.offline(opts.force)
elif (cmd == NGAMS_ONLINE_CMD):
stat = client.online()
elif (cmd == NGAMS_REARCHIVE_CMD):
if not opts.file_info_xml:
msg = "Must specify parameter -fileInfoXml for a REARCHIVE Command"
raise Exception(msg)
stat = client.rearchive(opts.file_uri, opts.file_info_xml, pars)
elif (cmd == NGAMS_REGISTER_CMD):
stat = client.register(opts.path, opts.asynchronous)
elif (cmd == NGAMS_REMDISK_CMD):
stat = client.remDisk(opts.disk_id, opts.execute)
elif (cmd == NGAMS_REMFILE_CMD):
stat = client.remFile(diskId=opts.disk_id,
fileId=opts.file_id,
fileVersion=opts.file_version,
execute=opts.execute)
elif (cmd == NGAMS_RETRIEVE_CMD):
stat = client.retrieve(opts.file_id,
opts.file_version,
pars=pars,
targetFile=opts.output,
processing=opts.p_plugin,
processingPars=opts.p_plugin_pars)
elif (cmd == NGAMS_STATUS_CMD):
stat = client.status(pars, opts.output)
elif (cmd == NGAMS_SUBSCRIBE_CMD):
if not opts.url:
raise Exception(
"Must specify parameter --url for a SUBSCRIBE commands")
stat = client.subscribe(url=opts.url,
priority=opts.priority,
startDate=opts.start_date,
filterPlugIn=opts.f_plugin,
filterPlugInPars=opts.f_plugin_pars,
pars=pars)
elif (cmd == NGAMS_UNSUBSCRIBE_CMD):
stat = client.unsubscribe(opts.url)
else:
stat = client.get_status(cmd, pars=pars)
if opts.verbose > 3 or stat.getStatus() == NGAMS_FAILURE:
printStatus(stat)
if opts.verbose > 4 and stat.getData():
print(stat.getData())
if opts.status:
xml = stat.genXml(0, 1, 1, 1).toprettyxml(' ', '\n')[0:-1]
if opts.status == DUMP_TO_STDOUT:
print(xml)
else:
with open(opts.status, 'w') as f:
f.write(xml)
if stat.getStatus() == NGAMS_FAILURE:
sys.exit(1)
def handleCmd(srvObj,
reqPropsObj,
httpRef):
"""
Handle STATUS command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler object
(ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
status = ngamsStatus.ngamsStatus()
status.\
setDate(toiso8601()).\
setVersion(getNgamsVersion()).setHostId(srvObj.getHostId()).\
setStatus(NGAMS_SUCCESS).\
setMessage("Successfully handled command STATUS").\
setState(srvObj.getState()).setSubState(srvObj.getSubState())
reqPropsObjRef = reqPropsObj
# Get the information requested.
diskId = ""
fileId = ""
fileVersion = ""
configurationFile = ""
fileAccess = ""
hostId = ""
requestId = ""
dbTime = ""
dbTimeReset = ""
fileList = ""
fileListId = ""
maxElements = 100000
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = reqPropsObj.getHttpPar("file_version")
if (reqPropsObj.hasHttpPar("configuration_file")): configurationFile = "-"
if (reqPropsObj.hasHttpPar("file_access")):
fileAccess = reqPropsObj.getHttpPar("file_access")
if (reqPropsObj.hasHttpPar("host_id")):
hostId = reqPropsObj.getHttpPar("host_id")
if (reqPropsObj.hasHttpPar("max_elements")):
maxElements = int(reqPropsObj.getHttpPar("max_elements"))
if (reqPropsObj.hasHttpPar("request_id")):
requestId = reqPropsObj.getHttpPar("request_id")
if (reqPropsObj.hasHttpPar("db_time")):
dbTime = True
if (reqPropsObj.hasHttpPar("db_time_reset")):
dbTimeReset = True
if (reqPropsObj.hasHttpPar("flush_log")):
# in the past this called flushLog()
# do we really want to keep that functionality? I doubt it
pass
if (reqPropsObj.hasHttpPar("file_list")):
fileList = int(reqPropsObj.getHttpPar("file_list"))
if (reqPropsObj.hasHttpPar("file_list_id")):
fileListId = reqPropsObj.getHttpPar("file_list_id")
if (reqPropsObj.hasHttpPar("dump_object_info")):
# Dump status of all objects allocated into file specified.
targFile = reqPropsObj.getHttpPar("dump_object_info")
rmFile(targFile)
fo = open(targFile, "w")
fo.write(_genObjectStatus())
fo.close()
# Handle request for file info.
genCfgStatus = 0
genDiskStatus = 0
genFileStatus = 0
genStatesStatus = 1
genRequestStatus = 0
msg = ""
help = 0
if (reqPropsObj.hasHttpPar("help")):
global _help
msg = _help
help = 1
elif hostId and hostId != srvObj.getHostId():
host, port = srvObj.get_remote_server_endpoint(hostId)
cfgObj = srvObj.getCfg()
if not cfgObj.getProxyMode():
httpRef.redirect(host, port)
return
else:
try:
httpRef.proxy_request(hostId, host, port)
except Exception, e:
ex = re.sub("<|>", "", str(e))
errMsg = genLog("NGAMS_ER_COM",
[host, port,ex])
raise Exception, errMsg
return