def createVolume(testParDic):
"""
Create the test 'archived' files according to the specified parameters.
testParDic: Dictionary with parameters for running the test (dictionary).
Returns: Void.
"""
info(4, "Entering createVolume() ...")
# Check that <Disk Mt Pt>/NgasDiskInfo is found. I.e., that this is an
# already registered NGAS Disk.
ngasDiskInfo = os.path.normpath("%s/NgasDiskInfo" %
testParDic["MOUNTPOINT"])
if (not os.path.exists(ngasDiskInfo)):
msg = "The speficied mount point: %s appears not to point to a " +\
"registered NGAS Disk"
raise Exception, msg % testParDic["MOUNTPOINT"]
# Check that basefile exists.
if (not os.path.exists(testParDic["BASEFILE"])):
msg = "The speficied basefile: %s appears not to exist"
raise Exception, msg % testParDic["BASEFILE"]
if (testParDic["BASEFILE"].find(".fits") != -1):
createFitsFiles(testParDic)
else:
raise Exception, "Unsupported file format!"
info(4, "Leaving createVolume()")
def execTest(testId, testParDic, dbmObj, srvNode, srvPort, testNode, testPort,
cmd, cmdPars):
"""
Execute a test +
testId: Test ID (string).
testParDic: Dictionary with parameters for running the test
(dictionary).
dbmObj: Test Report DBM (ngamsDbm).
srvNode: Server Host (string).
srvPort: Server Port (integer).
testNode: Test node name (string).
testPort: Test port number (integer).
cmd: Command (string).
cmdPars: List with parameters (list).
Returns: Status (integer/SUCCESS=0,FAILURE=1).
"""
info(4, "Entering execTest() ...")
stat, testInfo = sendCmd(testParDic, srvNode, srvPort, cmd, cmdPars)
addTestResult(testId, testParDic, dbmObj, srvNode, srvPort, testNode,
testPort, stat, testInfo)
info(4, "Leaving execTest()")
return stat
def test_status_host_id(testParDic, dbConObj, dbmObj, srvNode, srvPort,
testNode, testPort, pars):
"""
Execute STATUS?host_id test.
testParDic: Test parameter dictionary containing the parameters specified
for the test (dictionary).
dbConObj: NG/AMS DB object connected to the DB (ngamsDb).
dbmObj: NG/AMS DBM object associated to open DBM. This DBM contains
the contents for the tests. See also man-page for
ngamsUtils.ngamsCheckSystem (ngamsDbm).
srvNode: Server node name of server contacted for the test (string).
srvPort: Port of server node (integer).
testNode: (Sub-)node to be tested (string).
testPort: Port number of sub-node (integer).
pars: Parameters to pass on to System Check Plug-In (string).
Returns: Status of executing the test (integer/0=SUCCESS, 1=FAILURE).
"""
info(4, "Entering test_status_host_id() ...")
cmdPars = [["host_id", testNode]]
stat = execTest("test_status_host_id", testParDic, dbmObj, srvNode,
srvPort, testNode, testPort, NGAMS_STATUS_CMD, cmdPars)
info(4, "Leaving test_status_host_id()")
return stat
def sendCmd(testParDic, srvNode, srvPort, cmd, cmdPars):
"""
Send a command to the specified server.
testPar: Dictionary with parameters for running the test
(dictionary).
srvNode: Server node (string).
srvPort: Server port (integer).
cmd: Command (string).
cmdPars: List of parameters to send (list).
Returns: Tuple containing information about the test:
(<Status (0|1)>, <XML Status>|<Info>|None>
"""
info(4, "Entering sendCmd() ...")
try:
stat = ngamsPClient.ngamsPClient().setTimeOut(testParDic["TIMEOUT"]).\
sendCmdGen(srvNode, srvPort, cmd, pars=cmdPars)
except Exception, e:
return (TEST_FAILURE, str(e))
def __retrieveThread(optDic, taskCtrl, dummy):
"""
"""
info(4, "Entering retrieveThread(%s) ..." % getThreadName())
client = ngamsPClient.ngamsPClient().\
parseSrvList(optDic["servers"][NGAS_OPT_VAL])
while (1):
nextFileId = taskCtrl.getNextFileId()
if (not nextFileId): thread.exit()
nextFileId = nextFileId[:-1]
stat = client.retrieve2File(nextFileId, targetFile=nextFileId)
fileSize = getFileSize(nextFileId)
taskCtrl.incBytesRecv(fileSize)
info(1, "Next File ID: %s" % nextFileId)
def addTestResult(testId, testParDic, dbmObj, srvNode, srvPort, testNode,
testPort, status, statusInfo):
"""
Add/update a test result in the DBM Test Report.
testId: ID of the test (string).
testParDic: Dictionary with parameters for running the test
(dictionary).
dbmObj: Test report DBM (ngamsDbm).
srvNode: Server (contacted) host (string).
srvPort: Port of server (contacted) host (integer).
testNode: Host tested (string).
testPort: Port number for host tested (integer).
status: Status of test (SUCCESS|FAILURE).
statusInfo: Additional information in connection with the test. Could be
NG/AMS XML Status Document or some error message (string).
Returns: Void.
"""
info(4, "Entering addTestResult() ...")
key = "%s_%s_%s_%s_%s" % (testId, srvNode, str(srvPort), testNode,
str(testPort))
if (dbmObj.hasKey(key)):
testRes = dbmObj.get(key)
else:
testRes = [testId] + (TEST_STAT_INFO * [None])
testRes[TEST_TIME] = time.time()
if (not testRes[LAST_REP_TIME]): testRes[LAST_REP_TIME] = 0
testRes[SRV_HOST_ID] = srvNode
testRes[SRV_PORT_NO] = srvPort
testRes[HOST_ID] = testNode
testRes[PORT_NO] = testPort
testRes[TEST_STAT] = status
testRes[TEST_STAT_INFO] = statusInfo
dbmObj.add(key, testRes)
dbmObj.sync()
info(4, "Leaving addTestResult()")
def checkSystem(testParDic):
"""
Carry out a system check for the given system
testParDic: Dictionary with parameters for running the test (dictionary).
Returns: Void.
"""
info(4, "Entering serverLoop() ...")
checkCreatePath(testParDic["WORKING-DIR"])
srvList = testParDic["SERVERS"].split(",")
srvList.sort()
if (testParDic["IGNORE-SERVERS"]):
ignoreSrvList = testParDic["IGNORE-SERVERS"].split(",")
ignoreSrvList.sort()
else:
ignoreSrvList = []
domainList = testParDic["DOMAINS"].split(",")
domainList.sort()
dbSrv, db, user, password = ngasUtilsLib.getDbPars()
dbCon = ngamsDb.ngamsDb(dbSrv, db, user, password, 0)
srvStr = str(srvList)[1:-1].replace(",", "-").replace(" ", "").\
replace("'", "")
domStr = str(domainList)[1:-1].replace(",", "-").replace(" ", "").\
replace("'", "")
dbmName = os.path.normpath("%s/NGAS_SYS_CHECK_%s_%s_%s.bsddb" %\
(testParDic["WORKING-DIR"], os.environ["USER"],
srvStr, domStr))
dbmObj = ngamsDbm.ngamsDbm(dbmName, writePerm=1)
testParDic["_testTmpPath"] = os.path.normpath("%s/NGAS_SYS_CHECK_%.6f" %\
(testParDic["WORKING-DIR"],
time.time()))
os.system("mkdir -p %s" % testParDic["_testTmpPath"])
try:
serverLoop(testParDic, srvList, ignoreSrvList, domainList, dbCon,
dbmObj)
testParDic["_endTime"] = time.time()
dbmObj.sync()
os.system("rm -rf %s" % testParDic["_testTmpPath"])
except Exception, e:
dbmObj.sync()
os.system("rm -rf %s" % testParDic["_testTmpPath"])
raise Exception, e
def storeFile(hdrs, data, outDir):
"""
Store data contained in HTTP response.
hdrs: HTTP headers (string).
data: Data contained in HTTP response (string).
outDir: Requested output directory (string).
Returns: Name of target file (string).
"""
hdrDic = ngamsLib.httpMsgObj2Dic(hdrs)
tmpLine = hdrDic["content-disposition"]
outFile = string.split(string.split(tmpLine, ";")[1], "=")[1]
outFileCompl = os.path.normpath("%s/%s" % (outDir, outFile))
fo = open(outFileCompl, "w")
fo.write(data)
fo.close()
info(1, "Generated output file: %s" % outFileCompl)
def ngamsGLEAM_rmimgtar_filterpi(srvObj,
plugInPars,
filename,
fileId,
fileVersion = -1,
reqPropsObj = None):
"""
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
Returns: 0 if the file does not match, 1 if it matches the
conditions (integer/0|1).
"""
parts = fileId.split("_")
if (2 != len(parts)):
return 0
if (len(parts[0]) == 10 and "images.tar" == parts[1]):
info(3, "RMIMGTAR - Executing: " + sql_query)
res = srvObj.getDb().query2(sql_query, args=(fileId, fileVersion))
if not res:
return 0 # not even in the queue, do not remove it
status = res[0][0] # the first colummn at the first record
if (0 == int(status)):
#info("RMIMGTAR - Filter returns true {0}/{1}".format(fileId, fileVersion))
return 1 # untar is done successfully already
else:
#info("RMIMGTAR - Filter returns false {0}/{1}".format(fileId, fileVersion))
return 0 # untar is not done yet or done but with exceptions
else:
return 0
def ngamsGLEAM_rmimgtar_jobpi(srvObj, plugInPars, filename, fileId,
fileVersion, diskId):
"""
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).(e.g. scale_factor=4,threshold=1E-5)
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
Returns: the return code of the compression plugin (integer).
"""
archive_cmd = "%s -host %s -port 7777 -fileUri %s -cmd QARCHIVE -mimeType application/octet-stream " % (
archive_client, archive_host, filename)
re = commands.getstatusoutput(archive_cmd)
#if (os.WIFEXITED(re[0])):
if (0 == re[0]):
info(3, 'Successfully archived file to mwa-process01: %s' % filename)
work_dir = srvObj.getCfg().getRootDirectory() + '/tmp/'
try:
#ngamsDiscardCmd._discardFile(srvObj, diskId, fileId, fileVersion,
# execute=1, tmpFilePat=work_dir)
cmd2 = "curl --connect-timeout 5 http://store04.icrar.org:7777/DISCARD?file_id=%s\\&file_version=%d\\&disk_id=%s\\&execute=1" % (
fileId, fileVersion, diskId)
re2 = commands.getstatusoutput(cmd2)
if (0 == re2[0]):
return (0, 'OK')
else:
raise Exception(re2[1].split('\n')[-1])
except Exception, e1:
warning("\nFail to discard untar file: {0}/{1}: {2}\n".format(
fileId, fileVersion, str(e1)))
return (500, str(e1))
def execTest(optDic):
"""
Carry out the tool execution.
optDic: Dictionary containing the options (dictionary).
Returns: Void.
"""
info(4, "Entering execTest() ...")
if (optDic["HELP"][NGAS_OPT_VAL]):
print correctUsage()
sys.exit(0)
taskCtrl = taskControl()
taskCtrl.fileIds = open(optDic["FILE-LIST"][NGAS_OPT_VAL]).readlines()
random.shuffle(taskCtrl.fileIds)
thrHandleDic = {}
noOfThreads = int(optDic["THREADS"][NGAS_OPT_VAL])
for n in range(1, (noOfThreads + 1)):
threadId = "RETRIEVE-THREAD-" + str(n)
args = (optDic, taskCtrl, None)
info(4, "Starting Retrieve Sub-Thread: %s" % threadId)
thrHandleDic[n] = threading.Thread(None, retrieveThread, threadId,
args)
thrHandleDic[n].setDaemon(0)
for n in range(1, (noOfThreads + 1)):
thrHandleDic[n].start()
startTime = time.time()
# Wait for threads to finish
thrFinishCount = 0
while (thrFinishCount < int(optDic["threads"][NGAS_OPT_VAL])):
time.sleep(0.100)
for n in thrHandleDic.keys():
if (not thrHandleDic[n].isAlive()):
del thrHandleDic[n]
thrFinishCount += 1
stopTime = time.time()
statMsg = "Total time: %.3fs. Total rate: %.6f MB/s" %\
((stopTime - startTime),
((taskCtrl.bytesReceived / (1024.*1024))/(stopTime-startTime)))
info(1, statMsg)
info(4, "Leaving execTest()")
def retrieveThread(optDic, taskCtrl, dummy):
"""
"""
info(4, "Entering retrieveThread(%s) ..." % getThreadName())
client = testClient().parseSrvList(optDic["servers"][NGAS_OPT_VAL])
while (1):
nextFileId = taskCtrl.getNextFileId()
if (not nextFileId): thread.exit()
nextFileId = nextFileId[:-1]
info(1, "Next File ID: %s" % nextFileId)
try:
fileSize, fileObj = client.retrieveFileObj(nextFileId)
sizeRemain = fileSize
while (sizeRemain):
if (sizeRemain < BLOCK_SIZE):
reqSize = sizeRemain
else:
reqSize = BLOCK_SIZE
buf = fileObj.read(reqSize)
sizeRemain -= len(buf)
taskCtrl.incBytesRecv(fileSize)
except Exception, e:
error("Error retrieving file with ID: %s - skipping. Error: %s" %\
(nextFileId, str(e)))
def execute(optDic):
"""
Carry out the tool execution.
optDic: Dictionary containing the options (dictionary).
Returns: Void.
"""
info(4,"Entering execute() ...")
if (optDic["help"][NGAS_OPT_VAL]):
print correctUsage()
sys.exit(0)
# Submit the query.
client = ngamsPClient.ngamsPClient()
client.parseSrvList(optDic["SERVER"][NGAS_OPT_VAL])
parameters = [["query", urllib.quote(optDic["QUERY"][NGAS_OPT_VAL])]]
if (optDic["format"][NGAS_OPT_VAL]):
parameters.append(["format", optDic["FORMAT"][NGAS_OPT_VAL]])
if (optDic["reload"][NGAS_OPT_VAL]):
parameters.append(["reload", "1"])
startTime = time.time()
stat = client.sendCmdGen("", -1, "QUERY", pars=parameters)
stopTime = time.time()
print "# Status: %s" % stat.getStatus()
print "# Query: %s" % optDic["QUERY"][NGAS_OPT_VAL]
print "# Time: %.6fs\n" % (stopTime - startTime)
# Handle the response, dump the result to stdout in case of success.
if (stat.getStatus() == NGAMS_SUCCESS):
print stat.getData()
else:
msg = "Error occurred executing query: %s" % stat.getMessage()
raise Exception, msg
info(4,"Leaving execute()")
def ngamsGLEAM_uimg_jobpi(srvObj, plugInPars, filename, fileId, fileVersion,
diskId):
"""
srvObj: Reference to NG/AMS Server Object (ngamsServer).
plugInPars: Parameters to take into account for the plug-in
execution (string).(e.g. scale_factor=4,threshold=1E-5)
fileId: File ID for file to test (string).
filename: Filename of (complete) (string).
fileVersion: Version of file to test (integer).
Returns: the return code of the compression plugin (integer).
"""
"""
1. run the update_img_val.py to produce output at /tmp/out.fits
2. update CRC and replace the current file
3. clean up both outfilename
"""
# 1. execute the update function
outfilename = "{0}/{2}_{1}".format(tmp_path, fileId, fileVersion)
cmd = "{0} {1} {2} {3}".format(py, py_exec, filename, outfilename)
info(3, "IMUPDATE - Executing: " + cmd)
re = execCmd(cmd)
# 2. update CRC and replace the current file
progress = 0
try:
info(3,
'IMUPDATE - Executing: Getting crc from {0}'.format(outfilename))
crc = getFileCRC(outfilename)
info(3, "IMUPDATE - Executing: " + sql_crc)
srvObj.getDb().query2(sql_crc, args=(crc, fileId, fileVersion, diskId))
cmd_cp = "cp {0} {1}".format(outfilename, filename)
info(3, "IMUPDATE - Executing: " + cmd_cp)
execCmd(cmd_cp)
return (0, 'OK')
except Exception, ex:
msg = "Updating CRC and replacing file {0} failed: {1}".format(
outfilename, ex)
error(msg)
return (500, msg)
def ingestExtFiles(fileListFile, notifEmail):
"""
Ingest the list of files given in the file referred to.
The files are listed as:
<Disk ID> <File ID> [<File Version>]
...
fileListFile: Filename of file containing the file list (string).
notifEmail: List of email addresses to inform about the
execution of the discation procedure (string).
Returns: Void.
"""
fileInfoList = parseFileList(fileListFile)
# Open DB connection.
server, db, user, password = ngasUtilsLib.getDbPars()
dbCon = ngamsDb.ngamsDb(server, db, user, password, 0)
# Find distinct Disk IDs.
diskIdDic = {}
for fileInfo in fileInfoList:
diskIdDic[fileInfo[0]] = None
info(1, "Disk IDs: " + str(diskIdDic.keys()))
# Register disks referred to as external disks in the DB.
dbDiskDic = {}
for diskId in diskIdDic.keys():
if (not dbDiskDic.has_key(diskId)):
diskInfo = dbCon.getDiskInfoFromDiskId(diskId)
if (diskInfo != []):
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\
unpackSqlResult(diskInfo)
else:
# Create a new entry for that disk in the DB.
info(1,"Creating new entry for external disk with ID: " +\
str(diskId))
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\
setArchive("EXTERNAL").\
setDiskId(diskId).\
setLogicalName(diskId).\
setHostId("").\
setSlotId("").\
setMounted(0).\
setMountPoint("EXTERNAL").\
setNumberOfFiles(0).\
setAvailableMb(0).\
setBytesStored(0).\
setCompleted(1).\
setCompletionDateFromSecs(0).\
setType("EXTERNAL").\
setManufacturer("UNKNOWN").\
setInstallationDateFromSecs(time.time()).\
setChecksum(0).\
setTotalDiskWriteTime(0).\
setLastCheckFromSecs(0).\
setLastHostId("").\
setStorageSetId(diskId)
diskInfoObj.write(dbCon)
diskIdDic[diskId] = diskInfoObj
# Loop over the files and register them in the DB.
fileRegList = []
sys.stdout.write("Registering files ...")
fileCount = 0
for fileInfo in fileInfoList:
diskId = fileInfo[0]
fileId = fileInfo[1]
fileVer = fileInfo[2]
fileInfoObj = ngamsFileInfo.ngamsFileInfo().\
setDiskId(diskId).\
setFilename(fileId).\
setFileId(fileId).\
setFileVersion(fileVer).\
setFormat("").\
setFileSize(0).\
setUncompressedFileSize(0).\
setCompression("").\
setIngestionDateFromSecs(0).\
setIgnore(0).\
setChecksum("").\
setChecksumPlugIn("").\
setFileStatus(NGAMS_FILE_STATUS_OK).\
setCreationDateFromSecs(0).\
setTag("EXTERNAL")
fileInfoObj.write(getHostName(), dbCon, 0, 1)
fileRegList.append((diskId, fileId, fileVer))
time.sleep(0.050)
fileCount += 1
if ((fileCount % 10) == 0): sys.stdout.write(".")
sys.stdout.write("\n")
info(1, "Registered %d files" % fileCount)
report = genReport(fileRegList)
if (notifEmail):
ngasUtilsLib.sendEmail("ngasRegisterExtFiles: FILE REGISTRATION " +\
"REPORT", notifEmail, report)
cmd: Command (string).
cmdPars: List of parameters to send (list).
Returns: Tuple containing information about the test:
(<Status (0|1)>, <XML Status>|<Info>|None>
"""
info(4, "Entering sendCmd() ...")
try:
stat = ngamsPClient.ngamsPClient().setTimeOut(testParDic["TIMEOUT"]).\
sendCmdGen(srvNode, srvPort, cmd, pars=cmdPars)
except Exception, e:
return (TEST_FAILURE, str(e))
info(4, "Leaving sendCmd()")
if (stat.getStatus() == NGAMS_SUCCESS):
return (TEST_SUCCESS, stat.genXmlDoc())
else:
return (TEST_FAILURE, stat.genXmlDoc())
def execTest(testId, testParDic, dbmObj, srvNode, srvPort, testNode, testPort,
cmd, cmdPars):
"""
Execute a test +
testId: Test ID (string).
testParDic: Dictionary with parameters for running the test
(dictionary).
def serverLoop(testParDic, serverList, ignoreSrvList, domainList, dbCon,
dbmObj):
"""
Loop over the contact servers and execute the tests for each node
in the given domains, going through the contact node.
testParDic: Dictionary with parameters for running the test
(dictionary).
serverList: List of node/ports to contact for the tests (list).
ignoreSrvList: List of servers to ignore (list).
domainList: List of domains to take into account (list).
dbCon: DB connection object (ngamsDb).
dbmObj: DBM object (ngamsDbm).
Returns: Number of problems encountered (integer).
"""
info(4, "Entering serverLoop() ...")
problemCount = 0
# Get list of nodes.
sqlQuery = "SELECT host_id, srv_port FROM ngas_hosts " +\
"WHERE domain IN ('%s') " +\
"AND (srv_state='ONLINE' OR srv_suspended=1)"
sqlQuery = sqlQuery % str(testParDic["DOMAINS"]).strip()
nodeList = dbCon.query(sqlQuery)
for server in serverList:
srvNode, srvPort = server.split(":")
info(2, "Server: %s:%s" % (srvNode, str(srvPort)))
for testNode, testPort in nodeList[0]:
# Check if this server should be ignored.
if (ngamsLib.elInList(ignoreSrvList, "%s:%s" %\
(testNode, str(testPort)))):
continue
info(2, " - Test node: %s/port: %s" % (testNode, testPort))
for testCase in ["test_status_host_id", "test_retrieve_files"]:
info(2, " - Test: %s ..." % testCase)
stat = loadInvokeTest(testCase, testParDic, dbCon, dbmObj,
srvNode, srvPort, testNode, testPort,
None)
if (not stat):
info(2, " - Result: SUCCESS")
else:
info(2, " - Result: FAILURE")
problemCount += stat
# Now, loop again over the failing nodes to cross-check.
for n in range(3):
problemCount = 0
for nodeKey in dbmObj.keys():
testRes = dbmObj.get(nodeKey)
if (testRes[TEST_STAT] == TEST_FAILURE):
testCase = testRes[TEST_ID]
srvNode = testRes[SRV_HOST_ID]
srvPort = testRes[SRV_PORT_NO]
testNode = testRes[HOST_ID]
testPort = testRes[PORT_NO]
# Is this node online at all?
onlineNodeList = nodeList[0]
testNodePort = (testNode, testPort)
if (not ngamsLib.elInList(onlineNodeList, testNodePort)):
continue
# Ignore this node?
testNodePort = "%s:%s" % (testNode, str(testPort))
if (ngamsLib.elInList(ignoreSrvList, "%s" % testNodePort)):
continue
msg = "Retrying test case: %s on node: %s:%s using server: "+\
"%s:%s"
info(
2, msg %
(testCase, testNode, str(testPort), srvNode, str(srvPort)))
stat = loadInvokeTest(testCase, testParDic, dbCon, dbmObj,
srvNode, srvPort, testNode, testPort,
None)
if (stat): problemCount += 1
# Make a small sleep if problems were found and we want to retry.
if (problemCount and (n < 2)): time.sleep(10)
info(4, "Leaving serverLoop()")
return problemCount
elif (par.find("--SRCHOST") == 0):
srcHost = sys.argv[idx].split("=")[-1]
elif (par.find("--PATH") == 0):
path = sys.argv[idx].split("=")[-1]
elif (par.find("--OUTPUTDIR") == 0):
outputDir = sys.argv[idx].split("=")[-1]
elif (par == "--LIST"):
list = 1
else:
raise Exception, "Unknown parameter: %s" % parOrg
idx += 1
except Exception, e:
print "\nProblem executing the tool: %s\n" % str(e)
print correctUsage()
sys.exit(1)
if ((not host) and (not port) or (not path)):
print correctUsage()
raise Exception, "Incorrect/missing command line parameter(s)!"
if (not srcHost): srcHost = host
try:
if (not outputDir): outputDir = os.getcwd()
outputDir = os.path.normpath("%s/%s" % (outputDir, srcHost))
downloadFiles(host, port, srcHost, path, outputDir, list)
if (not list):
info(1, "Finished downloading specified path: %s" % path)
except Exception, e:
print "Problem encountered handling request:\n\n%s\n" % str(e)
sys.exit(1)
# EOF
testParDic["DB"] = sys.argv[idx][(sys.argv[idx].find("=") +
1):]
elif (par.find("--MOUNTPOINT") == 0):
testParDic["MOUNTPOINT"] = sys.argv[idx].split("=")[-1]
elif (par.find("--NOOFFILES") == 0):
testParDic["NOOFFILES"] = int(sys.argv[idx].split("=")[-1])
elif (par.find("--VERBOSE") == 0):
setLogCond(0, "", 0, "", int(sys.argv[idx].split("=")[-1]))
else:
raise Exception, "Unknown parameter: %s" % parOrg
idx += 1
except Exception, e:
print "\nProblem executing the tool: %s\n" % str(e)
print correctUsage()
sys.exit(1)
try:
if ((not testParDic["BASEFILE"]) or (not testParDic["MOUNTPOINT"])
or (not testParDic["NOOFFILES"])):
print correctUsage()
raise Exception, "Incorrect/missing command line parameter(s)!"
startTime = time.time()
createVolume(testParDic)
deltaTime = (time.time() - startTime)
timePerFile = (deltaTime / testParDic["NOOFFILES"])
info(1, "Time: %.3fs, %.3fs/file" % (deltaTime, timePerFile))
except Exception, e:
print "Problem encountered during execution:\n\n%s\n" % str(e)
sys.exit(1)
# EOF
def downloadFiles(host, port, srcHost, path, outputDir, list=0):
"""
Download the files matching the given pattern via the host specified,
from the host(s) indicated.
The files downloaded will be stored in a directory structure of the
form:
<Output Dir>/<Source Host>/<File Path>
I.e., the filepath of the file is preserved, and the source host name is
also kept.
host: Name of host to be contacted (string).
port: Port number of NG/AMS Server (integer).
srcHost: List of hosts to be contacted (<Host 1>,<Host 2>,...).
Can also be a single host, wildcards is allowed (string).
path: File pattern of file(s) to be retrieved. If this is a
directory, the files in the directory will be retrieved
(string).
outputDir: Output directory. If not given, the current working point
will be chosen (string).
list: If set to 1 and the path given is a directory, rather than
retrieving the file in the remote directory, the names of
the remote files are listed on stdout (integer/0|1).
Returns: Void.
"""
if (not list): commands.getstatusoutput("mkdir -p %s" % outputDir)
reply, msg, hdrs, data = ngamsLib.httpGet(host,
port,
NGAMS_RETRIEVE_CMD,
pars=[["internal", path],
["host_id", srcHost]],
timeOut=10)
if (isNgasXmlStatusDoc(data)):
stat = ngamsStatus.ngamsStatus().unpackXmlDoc(data, 1)
if (stat.getStatus() == NGAMS_FAILURE):
msg = "Problem handling request: %s" % stat.getMessage()
raise Exception, msg
else:
stat = None
if (data.find("<?xml version=") == -1):
if (not list):
storeFile(hdrs, data, outputDir)
else:
print path
return
# The reply is a file list. Retrieve the file in question.
remDir = stat.getFileListList()[0].getComment().split(": ")[-1]
remDir = os.path.normpath(remDir)
outputDir = os.path.normpath("%s/%s" % (outputDir, remDir.split("/")[-1]))
if (not list): commands.getstatusoutput("mkdir -p %s" % outputDir)
client = ngamsPClient.ngamsPClient()
for fileInfo in stat.getFileListList()[0].getFileInfoObjList():
# If we should only list the contents and if the entry is a file,
# we only display its information.
if (list):
print "%s %-8s %-8s %-10s %-16s %s" %\
(fileInfo.getPermissions(), fileInfo.getOwner(),
fileInfo.getGroup(), str(fileInfo.getFileSize()),
fileInfo.getModDate(), fileInfo.getFilename())
# If the entry is a file and we should only list, we go to the
# next item.
if (fileInfo.getPermissions()[0] != "d"): continue
# Handle/download the item.
fn = fileInfo.getFilename()
basename = os.path.basename(fn)
trgFile = os.path.normpath("%s/%s" % (outputDir, basename))
if (not list): info(1, "Requesting: %s:%s ..." % (srcHost, fn))
reply2, msg2, hdrs2, data2 = \
ngamsLib.httpGet(host, port, NGAMS_RETRIEVE_CMD,
pars=[["internal", fn],
["host_id", srcHost]], timeOut=10)
# If the remote object specified is a file, retrieve it, otherwise if a
# directory, call this function recursively to retrieve the contents of
# the sub-folder(s).
if (isNgasXmlStatusDoc(data2)):
stat2 = ngamsStatus.ngamsStatus().unpackXmlDoc(data2, 1)
if (stat2.getStatus() == NGAMS_FAILURE):
msg = "Problem handling request: %s" % stat2.getMessage()
raise Exception, msg
# It seems to be another File List, we retrieve the contents
# of this recursively.
if (not list): info(1, "Remote object found: %s" % fn)
downloadFiles(host, port, srcHost, fn, outputDir, list)
else:
# Just retrieve the file.
storeFile(hdrs2, data2, outputDir)
return
