def createVolume(testParDic): """ Create the test 'archived' files according to the specified parameters. testParDic: Dictionary with parameters for running the test (dictionary). Returns: Void. """ info(4, "Entering createVolume() ...") # Check that <Disk Mt Pt>/NgasDiskInfo is found. I.e., that this is an # already registered NGAS Disk. ngasDiskInfo = os.path.normpath("%s/NgasDiskInfo" % testParDic["MOUNTPOINT"]) if (not os.path.exists(ngasDiskInfo)): msg = "The speficied mount point: %s appears not to point to a " +\ "registered NGAS Disk" raise Exception, msg % testParDic["MOUNTPOINT"] # Check that basefile exists. if (not os.path.exists(testParDic["BASEFILE"])): msg = "The speficied basefile: %s appears not to exist" raise Exception, msg % testParDic["BASEFILE"] if (testParDic["BASEFILE"].find(".fits") != -1): createFitsFiles(testParDic) else: raise Exception, "Unsupported file format!" info(4, "Leaving createVolume()")
def execTest(testId, testParDic, dbmObj, srvNode, srvPort, testNode, testPort, cmd, cmdPars): """ Execute a test + testId: Test ID (string). testParDic: Dictionary with parameters for running the test (dictionary). dbmObj: Test Report DBM (ngamsDbm). srvNode: Server Host (string). srvPort: Server Port (integer). testNode: Test node name (string). testPort: Test port number (integer). cmd: Command (string). cmdPars: List with parameters (list). Returns: Status (integer/SUCCESS=0,FAILURE=1). """ info(4, "Entering execTest() ...") stat, testInfo = sendCmd(testParDic, srvNode, srvPort, cmd, cmdPars) addTestResult(testId, testParDic, dbmObj, srvNode, srvPort, testNode, testPort, stat, testInfo) info(4, "Leaving execTest()") return stat
def test_status_host_id(testParDic, dbConObj, dbmObj, srvNode, srvPort, testNode, testPort, pars): """ Execute STATUS?host_id test. testParDic: Test parameter dictionary containing the parameters specified for the test (dictionary). dbConObj: NG/AMS DB object connected to the DB (ngamsDb). dbmObj: NG/AMS DBM object associated to open DBM. This DBM contains the contents for the tests. See also man-page for ngamsUtils.ngamsCheckSystem (ngamsDbm). srvNode: Server node name of server contacted for the test (string). srvPort: Port of server node (integer). testNode: (Sub-)node to be tested (string). testPort: Port number of sub-node (integer). pars: Parameters to pass on to System Check Plug-In (string). Returns: Status of executing the test (integer/0=SUCCESS, 1=FAILURE). """ info(4, "Entering test_status_host_id() ...") cmdPars = [["host_id", testNode]] stat = execTest("test_status_host_id", testParDic, dbmObj, srvNode, srvPort, testNode, testPort, NGAMS_STATUS_CMD, cmdPars) info(4, "Leaving test_status_host_id()") return stat
def sendCmd(testParDic, srvNode, srvPort, cmd, cmdPars): """ Send a command to the specified server. testPar: Dictionary with parameters for running the test (dictionary). srvNode: Server node (string). srvPort: Server port (integer). cmd: Command (string). cmdPars: List of parameters to send (list). Returns: Tuple containing information about the test: (<Status (0|1)>, <XML Status>|<Info>|None> """ info(4, "Entering sendCmd() ...") try: stat = ngamsPClient.ngamsPClient().setTimeOut(testParDic["TIMEOUT"]).\ sendCmdGen(srvNode, srvPort, cmd, pars=cmdPars) except Exception, e: return (TEST_FAILURE, str(e))
def __retrieveThread(optDic, taskCtrl, dummy): """ """ info(4, "Entering retrieveThread(%s) ..." % getThreadName()) client = ngamsPClient.ngamsPClient().\ parseSrvList(optDic["servers"][NGAS_OPT_VAL]) while (1): nextFileId = taskCtrl.getNextFileId() if (not nextFileId): thread.exit() nextFileId = nextFileId[:-1] stat = client.retrieve2File(nextFileId, targetFile=nextFileId) fileSize = getFileSize(nextFileId) taskCtrl.incBytesRecv(fileSize) info(1, "Next File ID: %s" % nextFileId)
def addTestResult(testId, testParDic, dbmObj, srvNode, srvPort, testNode, testPort, status, statusInfo): """ Add/update a test result in the DBM Test Report. testId: ID of the test (string). testParDic: Dictionary with parameters for running the test (dictionary). dbmObj: Test report DBM (ngamsDbm). srvNode: Server (contacted) host (string). srvPort: Port of server (contacted) host (integer). testNode: Host tested (string). testPort: Port number for host tested (integer). status: Status of test (SUCCESS|FAILURE). statusInfo: Additional information in connection with the test. Could be NG/AMS XML Status Document or some error message (string). Returns: Void. """ info(4, "Entering addTestResult() ...") key = "%s_%s_%s_%s_%s" % (testId, srvNode, str(srvPort), testNode, str(testPort)) if (dbmObj.hasKey(key)): testRes = dbmObj.get(key) else: testRes = [testId] + (TEST_STAT_INFO * [None]) testRes[TEST_TIME] = time.time() if (not testRes[LAST_REP_TIME]): testRes[LAST_REP_TIME] = 0 testRes[SRV_HOST_ID] = srvNode testRes[SRV_PORT_NO] = srvPort testRes[HOST_ID] = testNode testRes[PORT_NO] = testPort testRes[TEST_STAT] = status testRes[TEST_STAT_INFO] = statusInfo dbmObj.add(key, testRes) dbmObj.sync() info(4, "Leaving addTestResult()")
def checkSystem(testParDic): """ Carry out a system check for the given system testParDic: Dictionary with parameters for running the test (dictionary). Returns: Void. """ info(4, "Entering serverLoop() ...") checkCreatePath(testParDic["WORKING-DIR"]) srvList = testParDic["SERVERS"].split(",") srvList.sort() if (testParDic["IGNORE-SERVERS"]): ignoreSrvList = testParDic["IGNORE-SERVERS"].split(",") ignoreSrvList.sort() else: ignoreSrvList = [] domainList = testParDic["DOMAINS"].split(",") domainList.sort() dbSrv, db, user, password = ngasUtilsLib.getDbPars() dbCon = ngamsDb.ngamsDb(dbSrv, db, user, password, 0) srvStr = str(srvList)[1:-1].replace(",", "-").replace(" ", "").\ replace("'", "") domStr = str(domainList)[1:-1].replace(",", "-").replace(" ", "").\ replace("'", "") dbmName = os.path.normpath("%s/NGAS_SYS_CHECK_%s_%s_%s.bsddb" %\ (testParDic["WORKING-DIR"], os.environ["USER"], srvStr, domStr)) dbmObj = ngamsDbm.ngamsDbm(dbmName, writePerm=1) testParDic["_testTmpPath"] = os.path.normpath("%s/NGAS_SYS_CHECK_%.6f" %\ (testParDic["WORKING-DIR"], time.time())) os.system("mkdir -p %s" % testParDic["_testTmpPath"]) try: serverLoop(testParDic, srvList, ignoreSrvList, domainList, dbCon, dbmObj) testParDic["_endTime"] = time.time() dbmObj.sync() os.system("rm -rf %s" % testParDic["_testTmpPath"]) except Exception, e: dbmObj.sync() os.system("rm -rf %s" % testParDic["_testTmpPath"]) raise Exception, e
def storeFile(hdrs, data, outDir): """ Store data contained in HTTP response. hdrs: HTTP headers (string). data: Data contained in HTTP response (string). outDir: Requested output directory (string). Returns: Name of target file (string). """ hdrDic = ngamsLib.httpMsgObj2Dic(hdrs) tmpLine = hdrDic["content-disposition"] outFile = string.split(string.split(tmpLine, ";")[1], "=")[1] outFileCompl = os.path.normpath("%s/%s" % (outDir, outFile)) fo = open(outFileCompl, "w") fo.write(data) fo.close() info(1, "Generated output file: %s" % outFileCompl)
def ngamsGLEAM_rmimgtar_filterpi(srvObj, plugInPars, filename, fileId, fileVersion = -1, reqPropsObj = None): """ srvObj: Reference to NG/AMS Server Object (ngamsServer). plugInPars: Parameters to take into account for the plug-in execution (string). fileId: File ID for file to test (string). filename: Filename of (complete) (string). fileVersion: Version of file to test (integer). reqPropsObj: NG/AMS request properties object (ngamsReqProps). Returns: 0 if the file does not match, 1 if it matches the conditions (integer/0|1). """ parts = fileId.split("_") if (2 != len(parts)): return 0 if (len(parts[0]) == 10 and "images.tar" == parts[1]): info(3, "RMIMGTAR - Executing: " + sql_query) res = srvObj.getDb().query2(sql_query, args=(fileId, fileVersion)) if not res: return 0 # not even in the queue, do not remove it status = res[0][0] # the first colummn at the first record if (0 == int(status)): #info("RMIMGTAR - Filter returns true {0}/{1}".format(fileId, fileVersion)) return 1 # untar is done successfully already else: #info("RMIMGTAR - Filter returns false {0}/{1}".format(fileId, fileVersion)) return 0 # untar is not done yet or done but with exceptions else: return 0
def ngamsGLEAM_rmimgtar_jobpi(srvObj, plugInPars, filename, fileId, fileVersion, diskId): """ srvObj: Reference to NG/AMS Server Object (ngamsServer). plugInPars: Parameters to take into account for the plug-in execution (string).(e.g. scale_factor=4,threshold=1E-5) fileId: File ID for file to test (string). filename: Filename of (complete) (string). fileVersion: Version of file to test (integer). Returns: the return code of the compression plugin (integer). """ archive_cmd = "%s -host %s -port 7777 -fileUri %s -cmd QARCHIVE -mimeType application/octet-stream " % ( archive_client, archive_host, filename) re = commands.getstatusoutput(archive_cmd) #if (os.WIFEXITED(re[0])): if (0 == re[0]): info(3, 'Successfully archived file to mwa-process01: %s' % filename) work_dir = srvObj.getCfg().getRootDirectory() + '/tmp/' try: #ngamsDiscardCmd._discardFile(srvObj, diskId, fileId, fileVersion, # execute=1, tmpFilePat=work_dir) cmd2 = "curl --connect-timeout 5 http://store04.icrar.org:7777/DISCARD?file_id=%s\\&file_version=%d\\&disk_id=%s\\&execute=1" % ( fileId, fileVersion, diskId) re2 = commands.getstatusoutput(cmd2) if (0 == re2[0]): return (0, 'OK') else: raise Exception(re2[1].split('\n')[-1]) except Exception, e1: warning("\nFail to discard untar file: {0}/{1}: {2}\n".format( fileId, fileVersion, str(e1))) return (500, str(e1))
def execTest(optDic): """ Carry out the tool execution. optDic: Dictionary containing the options (dictionary). Returns: Void. """ info(4, "Entering execTest() ...") if (optDic["HELP"][NGAS_OPT_VAL]): print correctUsage() sys.exit(0) taskCtrl = taskControl() taskCtrl.fileIds = open(optDic["FILE-LIST"][NGAS_OPT_VAL]).readlines() random.shuffle(taskCtrl.fileIds) thrHandleDic = {} noOfThreads = int(optDic["THREADS"][NGAS_OPT_VAL]) for n in range(1, (noOfThreads + 1)): threadId = "RETRIEVE-THREAD-" + str(n) args = (optDic, taskCtrl, None) info(4, "Starting Retrieve Sub-Thread: %s" % threadId) thrHandleDic[n] = threading.Thread(None, retrieveThread, threadId, args) thrHandleDic[n].setDaemon(0) for n in range(1, (noOfThreads + 1)): thrHandleDic[n].start() startTime = time.time() # Wait for threads to finish thrFinishCount = 0 while (thrFinishCount < int(optDic["threads"][NGAS_OPT_VAL])): time.sleep(0.100) for n in thrHandleDic.keys(): if (not thrHandleDic[n].isAlive()): del thrHandleDic[n] thrFinishCount += 1 stopTime = time.time() statMsg = "Total time: %.3fs. Total rate: %.6f MB/s" %\ ((stopTime - startTime), ((taskCtrl.bytesReceived / (1024.*1024))/(stopTime-startTime))) info(1, statMsg) info(4, "Leaving execTest()")
def retrieveThread(optDic, taskCtrl, dummy): """ """ info(4, "Entering retrieveThread(%s) ..." % getThreadName()) client = testClient().parseSrvList(optDic["servers"][NGAS_OPT_VAL]) while (1): nextFileId = taskCtrl.getNextFileId() if (not nextFileId): thread.exit() nextFileId = nextFileId[:-1] info(1, "Next File ID: %s" % nextFileId) try: fileSize, fileObj = client.retrieveFileObj(nextFileId) sizeRemain = fileSize while (sizeRemain): if (sizeRemain < BLOCK_SIZE): reqSize = sizeRemain else: reqSize = BLOCK_SIZE buf = fileObj.read(reqSize) sizeRemain -= len(buf) taskCtrl.incBytesRecv(fileSize) except Exception, e: error("Error retrieving file with ID: %s - skipping. Error: %s" %\ (nextFileId, str(e)))
def execute(optDic): """ Carry out the tool execution. optDic: Dictionary containing the options (dictionary). Returns: Void. """ info(4,"Entering execute() ...") if (optDic["help"][NGAS_OPT_VAL]): print correctUsage() sys.exit(0) # Submit the query. client = ngamsPClient.ngamsPClient() client.parseSrvList(optDic["SERVER"][NGAS_OPT_VAL]) parameters = [["query", urllib.quote(optDic["QUERY"][NGAS_OPT_VAL])]] if (optDic["format"][NGAS_OPT_VAL]): parameters.append(["format", optDic["FORMAT"][NGAS_OPT_VAL]]) if (optDic["reload"][NGAS_OPT_VAL]): parameters.append(["reload", "1"]) startTime = time.time() stat = client.sendCmdGen("", -1, "QUERY", pars=parameters) stopTime = time.time() print "# Status: %s" % stat.getStatus() print "# Query: %s" % optDic["QUERY"][NGAS_OPT_VAL] print "# Time: %.6fs\n" % (stopTime - startTime) # Handle the response, dump the result to stdout in case of success. if (stat.getStatus() == NGAMS_SUCCESS): print stat.getData() else: msg = "Error occurred executing query: %s" % stat.getMessage() raise Exception, msg info(4,"Leaving execute()")
def ngamsGLEAM_uimg_jobpi(srvObj, plugInPars, filename, fileId, fileVersion, diskId): """ srvObj: Reference to NG/AMS Server Object (ngamsServer). plugInPars: Parameters to take into account for the plug-in execution (string).(e.g. scale_factor=4,threshold=1E-5) fileId: File ID for file to test (string). filename: Filename of (complete) (string). fileVersion: Version of file to test (integer). Returns: the return code of the compression plugin (integer). """ """ 1. run the update_img_val.py to produce output at /tmp/out.fits 2. update CRC and replace the current file 3. clean up both outfilename """ # 1. execute the update function outfilename = "{0}/{2}_{1}".format(tmp_path, fileId, fileVersion) cmd = "{0} {1} {2} {3}".format(py, py_exec, filename, outfilename) info(3, "IMUPDATE - Executing: " + cmd) re = execCmd(cmd) # 2. update CRC and replace the current file progress = 0 try: info(3, 'IMUPDATE - Executing: Getting crc from {0}'.format(outfilename)) crc = getFileCRC(outfilename) info(3, "IMUPDATE - Executing: " + sql_crc) srvObj.getDb().query2(sql_crc, args=(crc, fileId, fileVersion, diskId)) cmd_cp = "cp {0} {1}".format(outfilename, filename) info(3, "IMUPDATE - Executing: " + cmd_cp) execCmd(cmd_cp) return (0, 'OK') except Exception, ex: msg = "Updating CRC and replacing file {0} failed: {1}".format( outfilename, ex) error(msg) return (500, msg)
def ingestExtFiles(fileListFile, notifEmail): """ Ingest the list of files given in the file referred to. The files are listed as: <Disk ID> <File ID> [<File Version>] ... fileListFile: Filename of file containing the file list (string). notifEmail: List of email addresses to inform about the execution of the discation procedure (string). Returns: Void. """ fileInfoList = parseFileList(fileListFile) # Open DB connection. server, db, user, password = ngasUtilsLib.getDbPars() dbCon = ngamsDb.ngamsDb(server, db, user, password, 0) # Find distinct Disk IDs. diskIdDic = {} for fileInfo in fileInfoList: diskIdDic[fileInfo[0]] = None info(1, "Disk IDs: " + str(diskIdDic.keys())) # Register disks referred to as external disks in the DB. dbDiskDic = {} for diskId in diskIdDic.keys(): if (not dbDiskDic.has_key(diskId)): diskInfo = dbCon.getDiskInfoFromDiskId(diskId) if (diskInfo != []): diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\ unpackSqlResult(diskInfo) else: # Create a new entry for that disk in the DB. info(1,"Creating new entry for external disk with ID: " +\ str(diskId)) diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\ setArchive("EXTERNAL").\ setDiskId(diskId).\ setLogicalName(diskId).\ setHostId("").\ setSlotId("").\ setMounted(0).\ setMountPoint("EXTERNAL").\ setNumberOfFiles(0).\ setAvailableMb(0).\ setBytesStored(0).\ setCompleted(1).\ setCompletionDateFromSecs(0).\ setType("EXTERNAL").\ setManufacturer("UNKNOWN").\ setInstallationDateFromSecs(time.time()).\ setChecksum(0).\ setTotalDiskWriteTime(0).\ setLastCheckFromSecs(0).\ setLastHostId("").\ setStorageSetId(diskId) diskInfoObj.write(dbCon) diskIdDic[diskId] = diskInfoObj # Loop over the files and register them in the DB. fileRegList = [] sys.stdout.write("Registering files ...") fileCount = 0 for fileInfo in fileInfoList: diskId = fileInfo[0] fileId = fileInfo[1] fileVer = fileInfo[2] fileInfoObj = ngamsFileInfo.ngamsFileInfo().\ setDiskId(diskId).\ setFilename(fileId).\ setFileId(fileId).\ setFileVersion(fileVer).\ setFormat("").\ setFileSize(0).\ setUncompressedFileSize(0).\ setCompression("").\ setIngestionDateFromSecs(0).\ setIgnore(0).\ setChecksum("").\ setChecksumPlugIn("").\ setFileStatus(NGAMS_FILE_STATUS_OK).\ setCreationDateFromSecs(0).\ setTag("EXTERNAL") fileInfoObj.write(getHostName(), dbCon, 0, 1) fileRegList.append((diskId, fileId, fileVer)) time.sleep(0.050) fileCount += 1 if ((fileCount % 10) == 0): sys.stdout.write(".") sys.stdout.write("\n") info(1, "Registered %d files" % fileCount) report = genReport(fileRegList) if (notifEmail): ngasUtilsLib.sendEmail("ngasRegisterExtFiles: FILE REGISTRATION " +\ "REPORT", notifEmail, report)
cmd: Command (string). cmdPars: List of parameters to send (list). Returns: Tuple containing information about the test: (<Status (0|1)>, <XML Status>|<Info>|None> """ info(4, "Entering sendCmd() ...") try: stat = ngamsPClient.ngamsPClient().setTimeOut(testParDic["TIMEOUT"]).\ sendCmdGen(srvNode, srvPort, cmd, pars=cmdPars) except Exception, e: return (TEST_FAILURE, str(e)) info(4, "Leaving sendCmd()") if (stat.getStatus() == NGAMS_SUCCESS): return (TEST_SUCCESS, stat.genXmlDoc()) else: return (TEST_FAILURE, stat.genXmlDoc()) def execTest(testId, testParDic, dbmObj, srvNode, srvPort, testNode, testPort, cmd, cmdPars): """ Execute a test + testId: Test ID (string). testParDic: Dictionary with parameters for running the test (dictionary).
def serverLoop(testParDic, serverList, ignoreSrvList, domainList, dbCon, dbmObj): """ Loop over the contact servers and execute the tests for each node in the given domains, going through the contact node. testParDic: Dictionary with parameters for running the test (dictionary). serverList: List of node/ports to contact for the tests (list). ignoreSrvList: List of servers to ignore (list). domainList: List of domains to take into account (list). dbCon: DB connection object (ngamsDb). dbmObj: DBM object (ngamsDbm). Returns: Number of problems encountered (integer). """ info(4, "Entering serverLoop() ...") problemCount = 0 # Get list of nodes. sqlQuery = "SELECT host_id, srv_port FROM ngas_hosts " +\ "WHERE domain IN ('%s') " +\ "AND (srv_state='ONLINE' OR srv_suspended=1)" sqlQuery = sqlQuery % str(testParDic["DOMAINS"]).strip() nodeList = dbCon.query(sqlQuery) for server in serverList: srvNode, srvPort = server.split(":") info(2, "Server: %s:%s" % (srvNode, str(srvPort))) for testNode, testPort in nodeList[0]: # Check if this server should be ignored. if (ngamsLib.elInList(ignoreSrvList, "%s:%s" %\ (testNode, str(testPort)))): continue info(2, " - Test node: %s/port: %s" % (testNode, testPort)) for testCase in ["test_status_host_id", "test_retrieve_files"]: info(2, " - Test: %s ..." % testCase) stat = loadInvokeTest(testCase, testParDic, dbCon, dbmObj, srvNode, srvPort, testNode, testPort, None) if (not stat): info(2, " - Result: SUCCESS") else: info(2, " - Result: FAILURE") problemCount += stat # Now, loop again over the failing nodes to cross-check. for n in range(3): problemCount = 0 for nodeKey in dbmObj.keys(): testRes = dbmObj.get(nodeKey) if (testRes[TEST_STAT] == TEST_FAILURE): testCase = testRes[TEST_ID] srvNode = testRes[SRV_HOST_ID] srvPort = testRes[SRV_PORT_NO] testNode = testRes[HOST_ID] testPort = testRes[PORT_NO] # Is this node online at all? onlineNodeList = nodeList[0] testNodePort = (testNode, testPort) if (not ngamsLib.elInList(onlineNodeList, testNodePort)): continue # Ignore this node? testNodePort = "%s:%s" % (testNode, str(testPort)) if (ngamsLib.elInList(ignoreSrvList, "%s" % testNodePort)): continue msg = "Retrying test case: %s on node: %s:%s using server: "+\ "%s:%s" info( 2, msg % (testCase, testNode, str(testPort), srvNode, str(srvPort))) stat = loadInvokeTest(testCase, testParDic, dbCon, dbmObj, srvNode, srvPort, testNode, testPort, None) if (stat): problemCount += 1 # Make a small sleep if problems were found and we want to retry. if (problemCount and (n < 2)): time.sleep(10) info(4, "Leaving serverLoop()") return problemCount
elif (par.find("--SRCHOST") == 0): srcHost = sys.argv[idx].split("=")[-1] elif (par.find("--PATH") == 0): path = sys.argv[idx].split("=")[-1] elif (par.find("--OUTPUTDIR") == 0): outputDir = sys.argv[idx].split("=")[-1] elif (par == "--LIST"): list = 1 else: raise Exception, "Unknown parameter: %s" % parOrg idx += 1 except Exception, e: print "\nProblem executing the tool: %s\n" % str(e) print correctUsage() sys.exit(1) if ((not host) and (not port) or (not path)): print correctUsage() raise Exception, "Incorrect/missing command line parameter(s)!" if (not srcHost): srcHost = host try: if (not outputDir): outputDir = os.getcwd() outputDir = os.path.normpath("%s/%s" % (outputDir, srcHost)) downloadFiles(host, port, srcHost, path, outputDir, list) if (not list): info(1, "Finished downloading specified path: %s" % path) except Exception, e: print "Problem encountered handling request:\n\n%s\n" % str(e) sys.exit(1) # EOF
testParDic["DB"] = sys.argv[idx][(sys.argv[idx].find("=") + 1):] elif (par.find("--MOUNTPOINT") == 0): testParDic["MOUNTPOINT"] = sys.argv[idx].split("=")[-1] elif (par.find("--NOOFFILES") == 0): testParDic["NOOFFILES"] = int(sys.argv[idx].split("=")[-1]) elif (par.find("--VERBOSE") == 0): setLogCond(0, "", 0, "", int(sys.argv[idx].split("=")[-1])) else: raise Exception, "Unknown parameter: %s" % parOrg idx += 1 except Exception, e: print "\nProblem executing the tool: %s\n" % str(e) print correctUsage() sys.exit(1) try: if ((not testParDic["BASEFILE"]) or (not testParDic["MOUNTPOINT"]) or (not testParDic["NOOFFILES"])): print correctUsage() raise Exception, "Incorrect/missing command line parameter(s)!" startTime = time.time() createVolume(testParDic) deltaTime = (time.time() - startTime) timePerFile = (deltaTime / testParDic["NOOFFILES"]) info(1, "Time: %.3fs, %.3fs/file" % (deltaTime, timePerFile)) except Exception, e: print "Problem encountered during execution:\n\n%s\n" % str(e) sys.exit(1) # EOF
def downloadFiles(host, port, srcHost, path, outputDir, list=0): """ Download the files matching the given pattern via the host specified, from the host(s) indicated. The files downloaded will be stored in a directory structure of the form: <Output Dir>/<Source Host>/<File Path> I.e., the filepath of the file is preserved, and the source host name is also kept. host: Name of host to be contacted (string). port: Port number of NG/AMS Server (integer). srcHost: List of hosts to be contacted (<Host 1>,<Host 2>,...). Can also be a single host, wildcards is allowed (string). path: File pattern of file(s) to be retrieved. If this is a directory, the files in the directory will be retrieved (string). outputDir: Output directory. If not given, the current working point will be chosen (string). list: If set to 1 and the path given is a directory, rather than retrieving the file in the remote directory, the names of the remote files are listed on stdout (integer/0|1). Returns: Void. """ if (not list): commands.getstatusoutput("mkdir -p %s" % outputDir) reply, msg, hdrs, data = ngamsLib.httpGet(host, port, NGAMS_RETRIEVE_CMD, pars=[["internal", path], ["host_id", srcHost]], timeOut=10) if (isNgasXmlStatusDoc(data)): stat = ngamsStatus.ngamsStatus().unpackXmlDoc(data, 1) if (stat.getStatus() == NGAMS_FAILURE): msg = "Problem handling request: %s" % stat.getMessage() raise Exception, msg else: stat = None if (data.find("<?xml version=") == -1): if (not list): storeFile(hdrs, data, outputDir) else: print path return # The reply is a file list. Retrieve the file in question. remDir = stat.getFileListList()[0].getComment().split(": ")[-1] remDir = os.path.normpath(remDir) outputDir = os.path.normpath("%s/%s" % (outputDir, remDir.split("/")[-1])) if (not list): commands.getstatusoutput("mkdir -p %s" % outputDir) client = ngamsPClient.ngamsPClient() for fileInfo in stat.getFileListList()[0].getFileInfoObjList(): # If we should only list the contents and if the entry is a file, # we only display its information. if (list): print "%s %-8s %-8s %-10s %-16s %s" %\ (fileInfo.getPermissions(), fileInfo.getOwner(), fileInfo.getGroup(), str(fileInfo.getFileSize()), fileInfo.getModDate(), fileInfo.getFilename()) # If the entry is a file and we should only list, we go to the # next item. if (fileInfo.getPermissions()[0] != "d"): continue # Handle/download the item. fn = fileInfo.getFilename() basename = os.path.basename(fn) trgFile = os.path.normpath("%s/%s" % (outputDir, basename)) if (not list): info(1, "Requesting: %s:%s ..." % (srcHost, fn)) reply2, msg2, hdrs2, data2 = \ ngamsLib.httpGet(host, port, NGAMS_RETRIEVE_CMD, pars=[["internal", fn], ["host_id", srcHost]], timeOut=10) # If the remote object specified is a file, retrieve it, otherwise if a # directory, call this function recursively to retrieve the contents of # the sub-folder(s). if (isNgasXmlStatusDoc(data2)): stat2 = ngamsStatus.ngamsStatus().unpackXmlDoc(data2, 1) if (stat2.getStatus() == NGAMS_FAILURE): msg = "Problem handling request: %s" % stat2.getMessage() raise Exception, msg # It seems to be another File List, we retrieve the contents # of this recursively. if (not list): info(1, "Remote object found: %s" % fn) downloadFiles(host, port, srcHost, fn, outputDir, list) else: # Just retrieve the file. storeFile(hdrs2, data2, outputDir) return