def try_archiving(cfg, srv, fname):
if not cfg.getArchiveRotatedLogfiles():
return
file_uri = "file://" + fname
host, port = srv.get_self_endpoint()
proto = srv.get_server_access_proto()
ngamsPClient.ngamsPClient(host, port,
proto=proto).archive(file_uri, 'ngas/nglog')
def test_Clone_2(self):
"""
Synopsis:
Send CLONE Command via P-Client.
Description:
Submit a CLONE Command via the P-Client. Disk ID, File ID and File
Version given. Do not wait for termination (wait=0).
Expected Result:
The CLONE Command should be send to the server and accepted.
Test Steps:
- Start server.
- Archive a FITS file.
- Submit a CLONE Command to clone the disk hosting the archived file.
- Check the reply from the server indicates that the Clone Request
was accepted for execution.
Remarks:
It is not checked if the files have been cloned as such. The CLONE
Command is tested in the Test Suite ngamsCloneCmdTest.py.
"""
self.prepExtSrv()
client = ngamsPClient.ngamsPClient(port=8888)
client.archive("src/SmallFile.fits")
statObj = client.clone("TEST.2001-05-08T15:25:00.123",
"tmp-ngamsTest-NGAS-FitsStorage1-Main-1",
-1,
async=1)
refMsg = "Accepted CLONE command for execution"
self.checkEqual(refMsg, statObj.getMessage(), "Problem executing " +\
"Archive Pull Request")
def test_Clone_1(self):
"""
Synopsis:
Send CLONE Command via P-Client.
Description:
Check the handling of CLONE Commands via the P-Client.
Expected Result:
The CLONE Command should be send to the server and should be
executed there as expected. Wait=1.
Test Steps:
- Start server.
- Archive a file.
- Submit a CLONE Command to clone the disk hosting the archived file
(wait=1).
- Check the response from the server.
Remarks:
It is not checked if the files have been cloned as such. The CLONE
Command is tested in the Test Suite ngamsCloneCmdTest.py.
"""
self.prepExtSrv()
client = ngamsPClient.ngamsPClient(port=8888)
client.archive("src/SmallFile.fits")
statObj = client.clone("TEST.2001-05-08T15:25:00.123",
"tmp-ngamsTest-NGAS-FitsStorage1-Main-1", -1)
refMsg = "Successfully handled command CLONE"
self.checkEqual(refMsg, statObj.getMessage(), "Problem executing " +\
"Archive Pull Request")
def test_Label_1(self):
"""
Synopsis:
Send LABEL Command via P-Client.
Description:
Test that it is possible to send LABEL Command via the P-Client.
Expected Result:
The LABEL Command should be send to the server and a label produced.
Test Steps:
- Start server.
- Submit LABEL Command specifying Slot ID/Host ID.
- Check response from the server.
Remarks:
...
"""
self.prepExtSrv()
status = ngamsPClient.ngamsPClient(port=8888).\
label("1", getHostName() + ":8888")
refMsg = "Successfully handled command LABEL"
self.checkEqual(refMsg, status.getMessage(), "Problem executing " +\
"LABEL Command")
def test_Register_2(self):
"""
Synopsis:
Send REGISTER Command via P-Client (wait=1).
Description:
Test that the REGISTER command is correctly handled via the NG/AMS
Python API.
Parameter wait specified = 1 (= there waited till command execution
has finished).
Expected Result:
The REGISTER Command should be send to the server, which should
start registering the files found under the given path. A response
should be returned when the execution finishes.
Test Steps:
- Start server.
- Copy file onto one the NGAS Disks.
- Send REGISTER Command via the P-Client to initiate the REGISTER
Command (wait=1).
- Check response from the server.
Remarks:
...
"""
self.prepExtSrv()
trgFile = "/tmp/ngamsTest/NGAS/FitsStorage1-Main-1/SmallFile.fits"
shutil.copy("src/SmallFile.fits", trgFile)
status = ngamsPClient.ngamsPClient(port=8888).\
register(trgFile)
refMsg = "Successfully handled command REGISTER"
self.checkEqual(refMsg, status.getMessage(), "Problem executing " +\
"REGISTER Command")
def test_Online_1(self):
"""
Synopsis:
Handling of ONLINE Command via P-Client.
Description:
The purpose of the test is to test the handling of ONLINE Commands
via the P-Client.
Expected Result:
The ONLINE Command should be send to the server by the P-Client.
The server should fo Online.
Test Steps:
- Start server (Auto Online=0).
- Check on response from server that command was successfully handled.
Remarks:
...
"""
self.prepExtSrv(autoOnline=0)
status = ngamsPClient.ngamsPClient(port=8888).online()
refMsg = "Successfully handled command ONLINE"
self.checkEqual(refMsg, status.getMessage(), "Problem executing " +\
"ONLINE Command")
def test_Archive_1(self):
"""
Synopsis:
Send Archive Push Request via P-Client.
Description:
Check the handling of Archive Push Requests through the NG/AMS
P-Client. No mime-type is submitted with the request.
Expected Result:
The ngamsPClient should accept the command and submit the Archive
Push Request to the remote NG/AMS Server.
Test Steps:
- Start server.
- Issue Archive Push Request (no mime-type).
- Wait for handling to finish.
- Check reply from server.
Remarks:
It is not checked if the file has been properly cloned. The ARCHIVE
Command is tested in the Test Suite ngamsArchiveCmdTest.py.
"""
self.prepExtSrv()
client = ngamsPClient.ngamsPClient(port=8888)
status = client.archive("src/SmallFile.fits")
refMsg = "Successfully handled Archive Push Request for data " +\
"file with URI: SmallFile.fits"
self.checkEqual(refMsg, status.getMessage(), "Problem executing " +\
"Archive Push Request")
def test_RemDisk_1(self):
"""
Synopsis:
Handling of REMDISK Command via P-Client (execute=1).
Description:
Test that the REMDISK command is correctly handled via the NG/AMS
Python API.
Expected Result:
The REMDISK Command should be send to the server and executed
accordingly.
Test Steps:
- Start server.
- Submit REMDISK Command to remove one of the registered disks
(execute=1).
- Check response from the server.
Remarks:
...
"""
self.prepExtSrv()
status = ngamsPClient.ngamsPClient(port=8888).\
remDisk("tmp-ngamsTest-NGAS-FitsStorage1-Main-1", 1)
refMsg = "NGAMS_INFO_DEL_DISK:4043:INFO: Successfully deleted " +\
"info for disk. Disk ID: " +\
"tmp-ngamsTest-NGAS-FitsStorage1-Main-1."
self.checkEqual(refMsg, status.getMessage(), "Problem executing " +\
"REMDISK Command")
def sendCmd(testParDic, srvNode, srvPort, cmd, cmdPars):
"""
Send a command to the specified server.
testPar: Dictionary with parameters for running the test
(dictionary).
srvNode: Server node (string).
srvPort: Server port (integer).
cmd: Command (string).
cmdPars: List of parameters to send (list).
Returns: Tuple containing information about the test:
(<Status (0|1)>, <XML Status>|<Info>|None>
"""
info(4, "Entering sendCmd() ...")
try:
stat = ngamsPClient.ngamsPClient().setTimeOut(testParDic["TIMEOUT"]).\
sendCmdGen(srvNode, srvPort, cmd, pars=cmdPars)
except Exception, e:
return (TEST_FAILURE, str(e))
def retrieveFiles(diskIdList):
"""
Retrieve all files associated to a list of Disk IDs.
diskIdList: List of Disk IDs (list).
Returns: Void.
"""
dbCon = ngamsDb.ngamsDb("ESOECF", "ngas", "ngas", "bmdhc19wdw==")
client = ngamsPClient.ngamsPClient("jewel1", 7777)
for diskId in diskIdList:
dbCur = dbCon.getFileInfoList("IBM-DTLA-307075-YSDYSG3X871")
while (1):
fileList = dbCur.fetch(100)
if (not fileList): break
for fileInfo in fileList:
fileId = fileInfo[ngamsDb.NGAS_FILES_FILE_ID]
fileVersion = fileInfo[ngamsDb.NGAS_FILES_FILE_VER]
print "Retrieving file with File ID/Version: %s/%d" %\
(fileId, fileVersion)
startTime = time.time()
res = client.retrieve(fileId, fileVersion,"/tmp/TestFile")
deltaTime = (time.time() - startTime)
if (res.getStatus() == NGAMS_SUCCESS):
format = " - Successfully retrieved file with " +\
"File ID/Version: %s/%d/Time: %.3fs"
print format % (fileId, fileVersion, deltaTime)
else:
format = " - ERROR retrieving file with " +\
"File ID/Version: %s/%d: %s/Time: %.3fs"
print format % (fileId, fileVersion,
str(res.getMessage()).replace("\n", " - "),
deltaTime)
rmFile("/tmp/TestFile")
def test_RemFile_1(self):
"""
Synopsis:
Send REMFILE Command via P-Client.
Description:
Test correct handling of the REMFILE command via the P-Client.
Expected Result:
The REMFILE Command should be send to the server, which should
execute it accordingly.
Test Steps:
- Start server.
- Archive a FITS file.
- Clone disk hosting the file.
- Issue REMFILE Command via the P-Client specifying to remove the
first copy of the file.
- Check the response from the server.
Remarks:
...
"""
self.prepExtSrv(cfgProps=(('NgamsCfg.Server[1].RequestDbBackend', 'memory'),))
client = ngamsPClient.ngamsPClient(port=8888)
client.archive("src/SmallFile.fits")
status = client.clone("", "tmp-ngamsTest-NGAS-FitsStorage1-Main-1", -1)
waitReqCompl(client, status.getRequestId())
status = client.remFile("tmp-ngamsTest-NGAS-FitsStorage1-Main-1",
"TEST.2001-05-08T15:25:00.123", -1)
refMsg = "NGAMS_INFO_FILE_DEL_STAT:4040:INFO: File deletion status. "+\
"Files Selected: 1, Files Deleted: 0, " +\
"Failed File Deletions: 0."
self.checkEqual(refMsg, status.getMessage(), "Problem executing " +\
"REMFILE Command")
def clone_check_file(disk_id, file_id, file_version, connection, ngas_client):
"""
Clone a file a check afterwards if it was successfully cloned
:param disk_id: ID of disk cloned (string)
:param file_id: ID of file cloned (string)
:param file_version: Version of file to check (integer)
:param connection: DB connection object (ngamsDb)
:param ngas_client: Initiated instance of NG/AMS P-Client Object (ngamsPClient)
:return: Updated Check Report (string)
"""
message = "\n-> Attempting to clone file: {:s}/{:s}/{:s}".format(
disk_id, file_id, file_version)
print(message)
message += " - status: "
result = ngas_client.clone(file_id, disk_id, file_version, wait=1)
if result.getStatus() == "FAILURE":
status = "FAILURE: " + str(result.get_message()) + "\n"
else:
# Check if file was really cloned
result = connection.getFileInfoFromFileIdHostId(
_get_target_host(), file_id, file_version)
if not result:
status = "FAILURE: File not cloned!\n"
else:
file_info = ngamsFileInfo.ngamsFileInfo().unpackSqlResult(result)
file_status_parameters = [["disk_id",
file_info.get_disk_id()],
["file_access", file_id],
["file_version", file_version]]
client = ngamsPClient.ngamsPClient(getHostName(),
ngas_client.getPort())
result = client.status(file_status_parameters)
status = result.get_message() + "\n"
return message + status
def test_Archive_2(self):
"""
Synopsis:
Send Archive Pull Request via P-Client
Description:
The purpose of the test is to test the handling of Archive Push
Requests via the NG/AMS P-Client.
Expected Result:
The P-Client should submit the Archive Pull Request to the
server, which should archive the file successfully.
Test Steps:
- Start server.
- Launch Archive Pull Request via the NG/AMS P-Client.
- Check the response from the server.
Remarks:
It is not checked if the file has been properly cloned. The ARCHIVE
Command is tested in the Test Suite ngamsArchiveCmdTest.py.
"""
self.prepExtSrv()
client = ngamsPClient.ngamsPClient(port=8888)
srcFileUri = "file:" + os.path.abspath("src/SmallFile.fits")
status = client.archive(srcFileUri)
refMsg = "Successfully handled Archive Pull Request for data file " +\
"with URI: " + srcFileUri
self.checkEqual(refMsg, status.getMessage(), "Problem executing " +\
"Archive Pull Request")
def test_Init_1(self):
"""
Synopsis:
Test that the INIT command is correctly handled via the NG/AMS
Python API.
Description:
Test that the INIT Command can be submitted via the P-Client.
Expected Result:
The INTI Command should be send to the server, which should accept
it and re-initialize.
Test Steps:
- Start server.
- Submit INIT Command.
- Check output from server that the command was successfully executed.
Remarks:
...
"""
self.prepExtSrv()
status = ngamsPClient.ngamsPClient(port=8888).init()
refMsg = "Successfully handled command INIT"
self.checkEqual(refMsg, status.getMessage(), "Problem executing " +\
"INIT Command")
def test_Clone_3(self):
"""
Synopsis:
Send CLONE Command via P-Client.
Description:
Test the CLONE Command. Disk ID, File ID and File Version given.
Wait for termination.
Expected Result:
The CLONE Command should be send to the server and accepted there.
Disk ID, File ID and File Version given. Wait for termination (wait=1).
Test Steps:
- Start server.
- Achive FITS file.
- Clone file specifying Disk ID, File ID and File Version + wait=1.
- Check response from server that the command was executed.
Remarks:
It is not checked if the files have been cloned as such. The CLONE
Command is tested in the Test Suite ngamsCloneCmdTest.py.
"""
self.prepExtSrv()
client = ngamsPClient.ngamsPClient(port=8888)
client.archive("src/SmallFile.fits")
statObj = client.clone("TEST.2001-05-08T15:25:00.123",
"tmp-ngamsTest-NGAS-FitsStorage1-Main-1",
-1,
async=0)
refMsg = "Successfully handled command CLONE"
self.checkEqual(refMsg, statObj.getMessage(), "Problem executing " +\
"Archive Pull Request")
def discard_files(file_list_file, execute, notification_email):
"""
Remove files from the NGAS DB + from from the disk. If files are given by their full path, only the file is
removed from disk, but the DB information remains.
:param file_list_file: Name of file containing list with references to files to remove (string)
:param execute: Actual remove the files (integer/0|1)
:param notification_email: List of email addresses to inform about the execution of the discard procedure (string)
"""
host = ngasUtilsLib.get_parameter_ngas_resource_file(ngasUtilsLib.NGAS_RC_PAR_HOST)
port = int(ngasUtilsLib.get_parameter_ngas_resource_file(ngasUtilsLib.NGAS_RC_PAR_PORT))
# client = ngamsPClient.ngamsPClient(host, port, auth="ngas:password")
client = ngamsPClient.ngamsPClient(host, port)
# Try to issue a DISCARD Command to the associated NG/AMS Server for each file listed in the File List
with open(file_list_file) as fo:
file_list_buffer = fo.readlines()
success_stat_list = []
failed_stat_list = []
for line in file_list_buffer:
line = line.strip()
if line == "" or line[0] == "#":
continue
if line[0] != "/":
# It's a "<Disk ID> <File ID> <File Version>" line
disk_id, file_id, file_version = _unpack_file_info(line)
parameters = [["disk_id", disk_id], ["file_id", file_id],
["file_version", file_version], ["execute", execute]]
else:
# The file is referred to by its complete path.
parameters = [["path", line], ["execute", execute]]
status = client.get_status(NGAMS_DISCARD_CMD, pars=parameters)
if status.getStatus() == NGAMS_SUCCESS:
success_stat_list.append((line, status))
else:
failed_stat_list.append((line, status))
# Generate report (Email Notification)
report = "FILE DISCARD REPORT:\n"
report += "Host: {:s}".format(getHostName())
if len(failed_stat_list):
if execute:
report += "\n\n=Failed File Discards:\n\n"
else:
report += "\n\n=Rejected File Discard Requests:\n\n"
for statInfo in failed_stat_list:
report += "{:s}: {:s}\n".format(statInfo[0], statInfo[1].getMessage())
if len(success_stat_list):
if execute:
report += "\n\n=Discarded Files:\n\n"
else:
report += "\n\n=Accepted Discard File Requests:\n\n"
for statInfo in success_stat_list:
report += "{:s}: {:s}\n".format(statInfo[0], statInfo[1].getMessage())
report += "\n# EOF\n"
print("\n" + report)
if notification_email:
ngasUtilsLib.send_email("ngasDiscardFiles: FILE DISCARD REPORT", notification_email, report)
def main():
opts = parseOptions()
if (not opts):
exit(1)
#pushUrl = opts.push_url
obsList = opts.obs_list.split(',')
host = opts.push_host
port = int(opts.port)
client = ngamsPClient.ngamsPClient(host, port, timeOut = NGAMS_SOCK_TIMEOUT_DEF)
toUrl = getPushURL("%s:%d" % (host, port), gateway = proxy_archive)
stageUrl = 'http://%s/ASYNCLISTRETRIEVE' % opts.data_mover
for obsNum in obsList:
print "Checking observation: %s" % obsNum
files = getFileIdsByObsNum(obsNum)
deliverFileIds = []
for fileId in files:
# first check if MIT has it or not
if (not hasMITGotIt(client, fileId)):
deliverFileIds.append(fileId)
"""
fileName = getFileFullPath(fileId)
if (not os.path.exists(fileName)):
print "\tFile %s does not exist" % fileName
continue
onTape = ngamsMWACortexTapeApi.isFileOnTape(fileName)
if (1 == onTape):
stageFile(fileName)
print "\tPushing file %s to MIT" % fileId
archiveFile(fileName, client)
"""
else:
print "\tFile %s is already at MIT. Skip it." % fileId
myReq = AsyncListRetrieveRequest(deliverFileIds, toUrl)
strReq = pickle.dumps(myReq)
try:
print "Sending async retrieve request to the data mover %s" % opts.data_mover
request = urllib2.Request(stageUrl)
base64string = base64.encodestring('ngasmgr:ngas$dba').replace('\n', '')
request.add_header("Authorization", "Basic %s" % base64string)
strRes = urllib2.urlopen(request, data = strReq, timeout = NGAMS_SOCK_TIMEOUT_DEF).read()
myRes = pickle.loads(strRes)
#strRes = urllib2.urlopen(stageUrl, data = strReq, timeout = NGAMS_SOCK_TIMEOUT_DEF).read()
#myRes = pickle.loads(strRes)
if (myRes):
print myRes.errorcode
else:
print 'Response is None when async staging files for obsNum %s' % obsNum
except (UnpicklingError, socket.timeout) as uerr:
print "Something wrong while sending async retrieve request for obsNum %s, %s" % (obsNum, str(uerr))
def __retrieveThread(optDic, taskCtrl, dummy):
"""
"""
info(4, "Entering retrieveThread(%s) ..." % getThreadName())
client = ngamsPClient.ngamsPClient().\
parseSrvList(optDic["servers"][NGAS_OPT_VAL])
while (1):
nextFileId = taskCtrl.getNextFileId()
if (not nextFileId): thread.exit()
nextFileId = nextFileId[:-1]
stat = client.retrieve2File(nextFileId, targetFile=nextFileId)
fileSize = getFileSize(nextFileId)
taskCtrl.incBytesRecv(fileSize)
info(1, "Next File ID: %s" % nextFileId)
def run(srvObj, stopEvt):
cfg = srvObj.getCfg()
logdir = os.path.dirname(cfg.getLocalLogFile())
# The LOG-ROTATE-<timestamp>.nglog.unsaved pattern is produced by the
# ngamsServer.NgasRotatingFileHandler class, which is the file handler
# attached to the root logger in the server process
logger.debug("Checking if there are unsaved rotated logfiles")
for unsaved in glob.glob(os.path.join(logdir,
'LOG-ROTATE-*.nglog.unsaved')):
# Remove the .unsaved bit, leave the rest
fname = '.'.join(unsaved.split('.')[:-1])
mvFile(unsaved, fname)
# Connect to the server and send a pull ARCHIVE request
if cfg.getArchiveRotatedLogfiles():
file_uri = "file://" + fname
host, port = srvObj.get_self_endpoint()
ngamsPClient.ngamsPClient(host,
port).archive(file_uri, 'ngas/nglog')
# Do additional things with our logfiles
for plugin in get_logfile_handler_plugins(cfg):
try:
plugin(srvObj, fname)
except:
logger.exception("Error while handling logfile %s", fname)
logger.debug("Check if there are old rotated logfiles to remove ...")
max_rotations = max(min(cfg.getLogRotateCache(), 100), 0)
logfiles = glob.glob(os.path.join(logdir, 'LOG-ROTATE-*.nglog'))
logfiles.sort()
for f in logfiles[max_rotations:]:
logger.info("Removing rotated logfile %s", f)
rmFile(f)
def test_ServerMultiplexing_01(self):
"""
Synopsis:
Test server multiplexing feature of the Python-API/Client.
Description:
The purpose of the test is to verify the proper functioning of the
server context switching (multiplexing) provided by the Python-
API/Client. With this feature a list of servers + ports is given, and
the C-API switches between the specified servers.
A STATUS Command will be sent out to test the server multiplexing.
Expected Result:
After a limited number of attempts, all servers available, should
have been contacted by the Python-Client.
Test Steps:
- Start simulated cluster with 5 units.
- Submit STATUS Command invoking the Python-Client with the list of
servers.
- Verify that within 100 attempts, all servers have been contacted.
Remarks:
...
Test Data:
...
"""
ports = range(8000, 8005)
hostname = getHostName()
self.prepCluster(ports)
srvList = [('127.0.0.1', p) for p in ports]
client = ngamsPClient.ngamsPClient(servers=srvList)
nodeDic = {"%s:%d" % (hostname, p): 0 for p in ports}
noOfNodes = len(nodeDic.keys())
nodeCount = 0
for n in range(100):
status = client.status()
if (nodeDic[status.getHostId()] == 0):
nodeDic[status.getHostId()] = 1
nodeCount += 1
if (nodeCount == noOfNodes): break
if (nodeCount != noOfNodes):
self.fail("Not all specified NGAS Nodes were contacted " +\
"within 100 attempts")
def check_file_list(host, port, file_list_file, notification_email,
ignore_disk_id):
"""
Check if each file in the list is accessible by sending a CHECKFILE Command to the specified NG/AMS Server
:param host: Host name of remote NG/AMS Server (string)
:param port: Port number used by remote NG/AMS Server (integer).
:param file_list_file: File containing list of Files IDs for files to check (string)
:param notification_email: Comma separated list of email recipients that should be informed about the actions
carried out, or the actions that would be carried out if executing the command (integer/0|1)
:param ignore_disk_id: Ignore the Disk ID of the file, just check that one such file is available (integer/0|1)
"""
file_reference_list = ngasUtilsLib.parse_file_list(file_list_file)
report = 50 * "=" + "\n"
report += "FILE CHECK REPORT:\n"
report += 50 * "-" + "\n"
print("\n" + report)
for file_info in file_reference_list:
disk_id = file_info[0]
file_id = file_info[1]
file_version = file_info[2]
if ignore_disk_id:
msg = "Checking file: {:s}/{:s} - Status: ".format(
str(file_id), str(file_version))
else:
msg = "Checking file: {:s}/{:s}/{:s} - Status: ".format(
str(disk_id), str(file_id), str(file_version))
print(msg)
# parameters = [["file_access", file_id], ["file_version", file_version]]
parameters = [["file_id", file_id], ["file_version", file_version]]
if not ignore_disk_id:
parameters.append(["disk_id", disk_id])
client = ngamsPClient.ngamsPClient(host, port)
# result = client.status(parameters)
# if result.getMessage().find("NGAMS_INFO_FILE_AVAIL") == -1:
result = client.get_status(NGAMS_CHECKFILE_CMD, pars=parameters)
if result.getMessage().find("FILE_OK") == -1:
status = NGAMS_FAILURE + "/" + result.getMessage()
else:
status = NGAMS_SUCCESS + "/OK"
print(status + "\n")
msg += status
report += msg + "\n"
report += 50 * "-" + "\n"
print(50 * "-" + "\n")
if notification_email:
ngasUtilsLib.send_email("ngasCheckFileList: FILE CHECK REPORT",
notification_email, report)
def run(srvObj, stopEvt):
"""
Check if there is enough disk space for the various
directories defined.
"""
try:
srvObj.checkDiskSpaceSat()
except Exception:
logger.exception("Not enough disk space, bringing the system Offline")
# Connect to the server and send an OFFLINE command
host, port = srvObj.get_self_endpoint()
auth = None
if srvObj.getCfg().getAuthorize():
auth = srvObj.getCfg().getAuthHttpHdrVal(NGAMS_HTTP_INT_AUTH_USER)
client = ngamsPClient.ngamsPClient(host, port, timeout=30, auth=auth)
client.offline(force=True)
def _scheduleQScanThread(srvObj, mrLocalTask):
global queScanThread
if (queScanThread == None or (not queScanThread.isAlive())):
if (not queScanThread):
logger.debug('queScanThread is None !!! Create it')
else:
logger.debug('queScanThread is Dead !!! Re-launch it')
ngas_hostId = srvObj.getHostId()
ngas_host = ngas_hostId.split(':')[0]
ngas_port = int(ngas_hostId.split(':')[1])
ngas_client = ngamsPClient.ngamsPClient(ngas_host, ngas_port)
jobManHost = srvObj.getCfg().getNGASJobMANHost()
args = (jobManHost, ngas_hostId, ngas_client)
queScanThread = threading.Thread(None, _queScanThread, 'QUESCAN_THRD',
args)
queScanThread.setDaemon(1)
queScanThread.start()
queTasks.put(mrLocalTask)
def test_Retrieve2File_1(self):
"""
Synopsis:
Handling of RETRIEVE Command via P-Client.
Description:
Test correct handling of the RETRIEVE command from the NG/AMS
Python API (no processing).
Expected Result:
The RETRIEVE Command should be send to the server requesting for
an archive file, which subsequently should be send back to the client
and stored on the local disk.
Test Steps:
- Start server.
- Archive a file.
- Retrieve the archived file into a local file.
- Check the response from the server.
- Check that the requested file has been successfully stored on the
local disk.
Remarks:
...
"""
self.prepExtSrv()
client = ngamsPClient.ngamsPClient(port=8888)
client.archive("src/SmallFile.fits")
trgDir = "tmp"
status = client.retrieve("TEST.2001-05-08T15:25:00.123",
targetFile=trgDir)
refMsg = "Successfully handled request"
self.checkEqual(refMsg, status.getMessage(), "Problem executing " +\
"RETRIEVE Command")
tmpFile = "tmp/TEST.2001-05-08T15:25:00.123.fits.gz"
unzipedTmpFile = 'tmp/SmallFile.fits'
unzip(tmpFile, unzipedTmpFile)
self.checkFilesEq("src/SmallFile.fits", unzipedTmpFile,
"Retrieved file incorrect")
def checkSrvRunning(host=None, port=7777):
"""
Function to check if server is running. It returns the status of the
server, which is one of the following values:
- NOT-RUNNING: Server is not running.
- OFFLINE: Server is running and is in OFFLINE State.
- ONLINE: Server is running and is in ONLINE State.
host: Host to check if server is running (string).
port: Port number used by server (integer).
Returns: Status (string).
"""
if host is None:
host = getHostName()
try:
res = ngamsPClient.ngamsPClient().\
sendCmdGen(host, port, NGAMS_STATUS_CMD)
status = res.getState()
except Exception, e:
status = NGAMS_NOT_RUN_STATE
def execute(optDic):
"""
Carry out the tool execution.
optDic: Dictionary containing the options (dictionary).
Returns: Void.
"""
info(4,"Entering execute() ...")
if (optDic["help"][NGAS_OPT_VAL]):
print correctUsage()
sys.exit(0)
# Submit the query.
client = ngamsPClient.ngamsPClient()
client.parseSrvList(optDic["SERVER"][NGAS_OPT_VAL])
parameters = [["query", urllib.quote(optDic["QUERY"][NGAS_OPT_VAL])]]
if (optDic["format"][NGAS_OPT_VAL]):
parameters.append(["format", optDic["FORMAT"][NGAS_OPT_VAL]])
if (optDic["reload"][NGAS_OPT_VAL]):
parameters.append(["reload", "1"])
startTime = time.time()
stat = client.sendCmdGen("", -1, "QUERY", pars=parameters)
stopTime = time.time()
print "# Status: %s" % stat.getStatus()
print "# Query: %s" % optDic["QUERY"][NGAS_OPT_VAL]
print "# Time: %.6fs\n" % (stopTime - startTime)
# Handle the response, dump the result to stdout in case of success.
if (stat.getStatus() == NGAMS_SUCCESS):
print stat.getData()
else:
msg = "Error occurred executing query: %s" % stat.getMessage()
raise Exception, msg
info(4,"Leaving execute()")
def _getClient(self):
from ngamsPClient import ngamsPClient
return ngamsPClient.ngamsPClient(self._host, self._port)
def _getClient(self):
from ngamsPClient import ngamsPClient
return ngamsPClient.ngamsPClient(self._host, self._port)
def meter_performance():
parser = argparse.ArgumentParser()
parser.add_argument('-H',
'--host',
help='The host where the server is running',
default='localhost')
parser.add_argument('-p',
'--port',
help='The port where the server is running',
type=int,
default=7777)
parser.add_argument('-t',
'--timeout',
help='Timeout used with server',
type=float,
default=60)
parser.add_argument('-n',
'--nret',
help='Number of retrievals to run in parallel',
type=int,
default=60)
parser.add_argument(
'-f',
'--file-id',
help='FileID to retrieve. Defaults to a random file in the server',
default=None)
parser.add_argument('-i',
'--iterations',
help='Iterations per thread',
type=int,
default=1)
parser.add_argument('-q',
'--quick',
help='Use quick_location=True',
action='store_true')
parser.add_argument('-v',
'--verbose',
help='Be more verbose',
action='count',
default=2)
opts = parser.parse_args()
levels = {
0: logging.CRITICAL,
1: logging.ERROR,
2: logging.WARNING,
3: logging.INFO,
4: logging.DEBUG,
5: logging.NOTSET
}
logging.basicConfig(level=levels[opts.verbose], stream=sys.stdout)
client = ngamsPClient.ngamsPClient(opts.host,
opts.port,
timeout=opts.timeout)
# Which file?
file_id = opts.file_id
if not file_id:
status = client.get_status('QUERY',
pars=(('query', 'files_list'), ('format',
'json')))
data = json.loads(status.getData())
file_id = data['files_list'][0]['col3']
logger.info("Using file_id = %s" % (file_id, ))
# How big it is?
status = client.status(pars=(('file_id', file_id), ))
fsize = int(status.getDiskStatusList()[0].getFileObjList()
[0].getUncompressedFileSize())
fsize_mb = fsize / 1024. / 1024.
# Go, go, go!
pool = multiprocessing.Pool(opts.nret)
start = time.time()
results = list(
pool.map(
functools.partial(retrieve, client, file_id, opts.quick,
opts.iterations), range(opts.nret)))
duration = time.time() - start
# Overall stats
n_files = sum(map(len, results))
logger.warning(
'Retrieved %d files of %.3f [MB] each in %.3f [s] at %.1f [MB/s]',
n_files, fsize_mb, duration, n_files * fsize_mb / duration)
def _getClient(self):
from ngamsPClient import ngamsPClient # @UnresolvedImport
return ngamsPClient.ngamsPClient(self._ngasSrv, self._ngasPort,
self._ngasTimeout)
def ngamsGLEAM_RRI_FilterPI(srvObj,
plugInPars,
filename,
fileId,
fileVersion=-1,
reqPropsObj=None):
"""
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: 0 if the file does not match, 1 if it matches the
conditions (integer/0|1).
"""
match = 0
fn, fext = os.path.splitext(fileId)
if (fext.lower() in file_ext
and # only send FITS files, no measurement sets
_shouldSend(fileId)
and # # only send files satisfying certain string pattern criteria
srvObj.getDb().isLastVersion(
fileId, fileVersion)): # only send the (known) latest version
parDic = []
pars = ""
if ((plugInPars != "") and (plugInPars != None)):
pars = plugInPars
elif (reqPropsObj != None):
if (reqPropsObj.hasHttpPar("plug_in_pars")):
pars = reqPropsObj.getHttpPar("plug_in_pars")
parDic = ngamsPlugInApi.parseRawPlugInPars(pars)
if (not parDic.has_key("remote_host")
or not parDic.has_key("remote_port")):
errMsg = "ngamsGLEAM_VUW_FilterPI: Missing Plug-In Parameter: " +\
"remote_host / remote_port"
#raise Exception, errMsg
logger.error(errMsg)
return 1 # matched as if the remote checking is done
host = parDic["remote_host"]
sport = parDic["remote_port"]
if (not sport.isdigit()):
errMsg = "ngamsGLEAM_VUW_FilterPI: Invalid port number: " + sport
logger.error(errMsg)
return 1 # matched as if the filter does not exist
port = int(sport)
# Perform the matching.
client = ngamsPClient.ngamsPClient(host,
port,
timeOut=NGAMS_SOCK_TIMEOUT_DEF)
try:
if (fileVersion == -1):
fileVersion = 1
rest = client.get_status(NGAMS_STATUS_CMD,
pars=[["file_id", fileId],
["file_version", fileVersion]])
# since the queue will be sorted based on ingestion date, this will ensure the versions are sent by order:
# e.g. version1, version2, version3, otherwise, this method will have disordered versions sent
if (rest.getStatus().find(NGAMS_FAILURE) != -1):
return 1 # matched since file id does not exist
except Exception as e:
errMsg = "Error occurred during checking remote file status " +\
"ngamsGLEAM_VUW_FilterPI. Exception: " + str(e)
logger.error(errMsg)
return 1 # matched as if the filter does not exist
#info(5, "filter return status = " + rest.getStatus())
#info(4, "filter match = " + str(match))
return match
def _getClient(self):
from ngamsPClient import ngamsPClient # @UnresolvedImport
return ngamsPClient.ngamsPClient(self._ngasSrv, self._ngasPort, self._ngasTimeout)