Example #1
0
def createVolume(testParDic):
    """
    Create the test 'archived' files according to the specified parameters.

    testParDic:  Dictionary with parameters for running the test (dictionary).

    Returns:     Void.
    """
    info(4, "Entering createVolume() ...")

    # Check that <Disk Mt Pt>/NgasDiskInfo is found. I.e., that this is an
    # already registered NGAS Disk.
    ngasDiskInfo = os.path.normpath("%s/NgasDiskInfo" %
                                    testParDic["MOUNTPOINT"])
    if (not os.path.exists(ngasDiskInfo)):
        msg = "The speficied mount point: %s appears not to point to a " +\
              "registered NGAS Disk"
        raise Exception, msg % testParDic["MOUNTPOINT"]

    # Check that basefile exists.
    if (not os.path.exists(testParDic["BASEFILE"])):
        msg = "The speficied basefile: %s appears not to exist"
        raise Exception, msg % testParDic["BASEFILE"]

    if (testParDic["BASEFILE"].find(".fits") != -1):
        createFitsFiles(testParDic)
    else:
        raise Exception, "Unsupported file format!"
    info(4, "Leaving createVolume()")
Example #2
0
def execTest(testId, testParDic, dbmObj, srvNode, srvPort, testNode, testPort,
             cmd, cmdPars):
    """
    Execute a test +

    testId:     Test ID (string).

    testParDic: Dictionary with parameters for running the test
                (dictionary).

    dbmObj:     Test Report DBM (ngamsDbm).

    srvNode:    Server Host (string).

    srvPort:    Server Port (integer).

    testNode:   Test node name (string).

    testPort:   Test port number (integer).

    cmd:        Command (string).

    cmdPars:    List with parameters (list).

    Returns:    Status (integer/SUCCESS=0,FAILURE=1).
    """
    info(4, "Entering execTest() ...")
    stat, testInfo = sendCmd(testParDic, srvNode, srvPort, cmd, cmdPars)
    addTestResult(testId, testParDic, dbmObj, srvNode, srvPort, testNode,
                  testPort, stat, testInfo)
    info(4, "Leaving execTest()")
    return stat
Example #3
0
def test_status_host_id(testParDic, dbConObj, dbmObj, srvNode, srvPort,
                        testNode, testPort, pars):
    """
    Execute STATUS?host_id test.

    testParDic:   Test parameter dictionary containing the parameters specified
                  for the test (dictionary).

    dbConObj:     NG/AMS DB object connected to the DB (ngamsDb).

    dbmObj:       NG/AMS DBM object associated to open DBM. This DBM contains
                  the contents for the tests. See also man-page for
                  ngamsUtils.ngamsCheckSystem (ngamsDbm).

    srvNode:      Server node name of server contacted for the test (string).

    srvPort:      Port of server node (integer).

    testNode:     (Sub-)node to be tested (string).

    testPort:     Port number of sub-node (integer).

    pars:         Parameters to pass on to System Check Plug-In (string).

    Returns:      Status of executing the test (integer/0=SUCCESS, 1=FAILURE).
    """
    info(4, "Entering test_status_host_id() ...")
    cmdPars = [["host_id", testNode]]
    stat = execTest("test_status_host_id", testParDic, dbmObj, srvNode,
                    srvPort, testNode, testPort, NGAMS_STATUS_CMD, cmdPars)
    info(4, "Leaving test_status_host_id()")
    return stat
Example #4
0
def sendCmd(testParDic, srvNode, srvPort, cmd, cmdPars):
    """
    Send a command to the specified server.

    testPar:  Dictionary with parameters for running the test
              (dictionary).

    srvNode:  Server node (string).

    srvPort:  Server port (integer).

    cmd:      Command (string).

    cmdPars:  List of parameters to send (list).

    Returns:  Tuple containing information about the test:

                (<Status (0|1)>, <XML Status>|<Info>|None>
    """
    info(4, "Entering sendCmd() ...")
    try:
        stat = ngamsPClient.ngamsPClient().setTimeOut(testParDic["TIMEOUT"]).\
               sendCmdGen(srvNode, srvPort, cmd, pars=cmdPars)
    except Exception, e:
        return (TEST_FAILURE, str(e))
Example #5
0
def __retrieveThread(optDic, taskCtrl, dummy):
    """
    """
    info(4, "Entering retrieveThread(%s) ..." % getThreadName())
    client = ngamsPClient.ngamsPClient().\
             parseSrvList(optDic["servers"][NGAS_OPT_VAL])
    while (1):
        nextFileId = taskCtrl.getNextFileId()
        if (not nextFileId): thread.exit()
        nextFileId = nextFileId[:-1]
        stat = client.retrieve2File(nextFileId, targetFile=nextFileId)
        fileSize = getFileSize(nextFileId)
        taskCtrl.incBytesRecv(fileSize)
        info(1, "Next File ID: %s" % nextFileId)
Example #6
0
def addTestResult(testId, testParDic, dbmObj, srvNode, srvPort, testNode,
                  testPort, status, statusInfo):
    """
    Add/update a test result in the DBM Test Report.

    testId:      ID of the test (string).

    testParDic:  Dictionary with parameters for running the test
                 (dictionary).

    dbmObj:      Test report DBM (ngamsDbm).

    srvNode:     Server (contacted) host (string).

    srvPort:     Port of server (contacted) host (integer).

    testNode:    Host tested (string).

    testPort:    Port number for host tested (integer).

    status:      Status of test (SUCCESS|FAILURE).

    statusInfo:  Additional information in connection with the test. Could be
                 NG/AMS XML Status Document or some error message (string).

    Returns:     Void.
    """
    info(4, "Entering addTestResult() ...")
    key = "%s_%s_%s_%s_%s" % (testId, srvNode, str(srvPort), testNode,
                              str(testPort))
    if (dbmObj.hasKey(key)):
        testRes = dbmObj.get(key)
    else:
        testRes = [testId] + (TEST_STAT_INFO * [None])
    testRes[TEST_TIME] = time.time()
    if (not testRes[LAST_REP_TIME]): testRes[LAST_REP_TIME] = 0
    testRes[SRV_HOST_ID] = srvNode
    testRes[SRV_PORT_NO] = srvPort
    testRes[HOST_ID] = testNode
    testRes[PORT_NO] = testPort
    testRes[TEST_STAT] = status
    testRes[TEST_STAT_INFO] = statusInfo
    dbmObj.add(key, testRes)
    dbmObj.sync()
    info(4, "Leaving addTestResult()")
Example #7
0
def checkSystem(testParDic):
    """
    Carry out a system check for the given system

    testParDic:   Dictionary with parameters for running the test (dictionary).

    Returns:      Void.
    """
    info(4, "Entering serverLoop() ...")
    checkCreatePath(testParDic["WORKING-DIR"])
    srvList = testParDic["SERVERS"].split(",")
    srvList.sort()
    if (testParDic["IGNORE-SERVERS"]):
        ignoreSrvList = testParDic["IGNORE-SERVERS"].split(",")
        ignoreSrvList.sort()
    else:
        ignoreSrvList = []
    domainList = testParDic["DOMAINS"].split(",")
    domainList.sort()
    dbSrv, db, user, password = ngasUtilsLib.getDbPars()
    dbCon = ngamsDb.ngamsDb(dbSrv, db, user, password, 0)
    srvStr = str(srvList)[1:-1].replace(",", "-").replace(" ", "").\
             replace("'", "")
    domStr = str(domainList)[1:-1].replace(",", "-").replace(" ", "").\
             replace("'", "")
    dbmName = os.path.normpath("%s/NGAS_SYS_CHECK_%s_%s_%s.bsddb" %\
                               (testParDic["WORKING-DIR"], os.environ["USER"],
                                srvStr, domStr))
    dbmObj = ngamsDbm.ngamsDbm(dbmName, writePerm=1)
    testParDic["_testTmpPath"] = os.path.normpath("%s/NGAS_SYS_CHECK_%.6f" %\
                                                  (testParDic["WORKING-DIR"],
                                                   time.time()))
    os.system("mkdir -p %s" % testParDic["_testTmpPath"])
    try:
        serverLoop(testParDic, srvList, ignoreSrvList, domainList, dbCon,
                   dbmObj)
        testParDic["_endTime"] = time.time()
        dbmObj.sync()
        os.system("rm -rf %s" % testParDic["_testTmpPath"])
    except Exception, e:
        dbmObj.sync()
        os.system("rm -rf %s" % testParDic["_testTmpPath"])
        raise Exception, e
Example #8
0
def storeFile(hdrs, data, outDir):
    """
    Store data contained in HTTP response.

    hdrs:       HTTP headers (string).

    data:       Data contained in HTTP response (string).

    outDir:     Requested output directory (string).

    Returns:    Name of target file (string).
    """
    hdrDic = ngamsLib.httpMsgObj2Dic(hdrs)
    tmpLine = hdrDic["content-disposition"]
    outFile = string.split(string.split(tmpLine, ";")[1], "=")[1]
    outFileCompl = os.path.normpath("%s/%s" % (outDir, outFile))
    fo = open(outFileCompl, "w")
    fo.write(data)
    fo.close()
    info(1, "Generated output file: %s" % outFileCompl)
Example #9
0
def ngamsGLEAM_rmimgtar_filterpi(srvObj,
                          plugInPars,
                          filename,
                          fileId,
                          fileVersion = -1,
                          reqPropsObj = None):

    """
    srvObj:        Reference to NG/AMS Server Object (ngamsServer).

    plugInPars:    Parameters to take into account for the plug-in
                   execution (string).

    fileId:        File ID for file to test (string).

    filename:      Filename of (complete) (string).

    fileVersion:   Version of file to test (integer).

    reqPropsObj:   NG/AMS request properties object (ngamsReqProps).

    Returns:       0 if the file does not match, 1 if it matches the
                   conditions (integer/0|1).
    """
    parts = fileId.split("_")
    if (2 != len(parts)):
        return 0
    if (len(parts[0]) == 10 and "images.tar" == parts[1]):
        info(3, "RMIMGTAR - Executing: " + sql_query)
        res = srvObj.getDb().query2(sql_query, args=(fileId, fileVersion))
        if not res:
            return 0 # not even in the queue, do not remove it
        status = res[0][0] # the first colummn at the first record
        if (0 == int(status)):
            #info("RMIMGTAR - Filter returns true {0}/{1}".format(fileId, fileVersion))
            return 1 # untar is done successfully already
        else:
            #info("RMIMGTAR - Filter returns false {0}/{1}".format(fileId, fileVersion))
            return 0 # untar is not done yet or done but with exceptions
    else:
        return 0
Example #10
0
def ngamsGLEAM_rmimgtar_jobpi(srvObj, plugInPars, filename, fileId,
                              fileVersion, diskId):
    """
    srvObj:        Reference to NG/AMS Server Object (ngamsServer).

    plugInPars:    Parameters to take into account for the plug-in
                   execution (string).(e.g. scale_factor=4,threshold=1E-5)

    fileId:        File ID for file to test (string).

    filename:      Filename of (complete) (string).

    fileVersion:   Version of file to test (integer).

    Returns:       the return code of the compression plugin (integer).
    """
    archive_cmd = "%s -host %s -port 7777 -fileUri %s -cmd QARCHIVE -mimeType application/octet-stream " % (
        archive_client, archive_host, filename)
    re = commands.getstatusoutput(archive_cmd)
    #if (os.WIFEXITED(re[0])):
    if (0 == re[0]):
        info(3, 'Successfully archived file to mwa-process01: %s' % filename)
        work_dir = srvObj.getCfg().getRootDirectory() + '/tmp/'
        try:
            #ngamsDiscardCmd._discardFile(srvObj, diskId, fileId, fileVersion,
            #                             execute=1, tmpFilePat=work_dir)
            cmd2 = "curl --connect-timeout 5 http://store04.icrar.org:7777/DISCARD?file_id=%s\\&file_version=%d\\&disk_id=%s\\&execute=1" % (
                fileId, fileVersion, diskId)
            re2 = commands.getstatusoutput(cmd2)
            if (0 == re2[0]):
                return (0, 'OK')
            else:
                raise Exception(re2[1].split('\n')[-1])
        except Exception, e1:
            warning("\nFail to discard untar file: {0}/{1}: {2}\n".format(
                fileId, fileVersion, str(e1)))
            return (500, str(e1))
Example #11
0
def execTest(optDic):
    """
    Carry out the tool execution.

    optDic:    Dictionary containing the options (dictionary).

    Returns:   Void.
    """
    info(4, "Entering execTest() ...")
    if (optDic["HELP"][NGAS_OPT_VAL]):
        print correctUsage()
        sys.exit(0)

    taskCtrl = taskControl()
    taskCtrl.fileIds = open(optDic["FILE-LIST"][NGAS_OPT_VAL]).readlines()
    random.shuffle(taskCtrl.fileIds)
    thrHandleDic = {}
    noOfThreads = int(optDic["THREADS"][NGAS_OPT_VAL])
    for n in range(1, (noOfThreads + 1)):
        threadId = "RETRIEVE-THREAD-" + str(n)
        args = (optDic, taskCtrl, None)
        info(4, "Starting Retrieve Sub-Thread: %s" % threadId)
        thrHandleDic[n] = threading.Thread(None, retrieveThread, threadId,
                                           args)
        thrHandleDic[n].setDaemon(0)
    for n in range(1, (noOfThreads + 1)):
        thrHandleDic[n].start()

    startTime = time.time()

    # Wait for threads to finish
    thrFinishCount = 0
    while (thrFinishCount < int(optDic["threads"][NGAS_OPT_VAL])):
        time.sleep(0.100)
        for n in thrHandleDic.keys():
            if (not thrHandleDic[n].isAlive()):
                del thrHandleDic[n]
                thrFinishCount += 1

    stopTime = time.time()
    statMsg = "Total time: %.3fs. Total rate: %.6f MB/s" %\
              ((stopTime - startTime),
               ((taskCtrl.bytesReceived / (1024.*1024))/(stopTime-startTime)))
    info(1, statMsg)

    info(4, "Leaving execTest()")
Example #12
0
def retrieveThread(optDic, taskCtrl, dummy):
    """
    """
    info(4, "Entering retrieveThread(%s) ..." % getThreadName())
    client = testClient().parseSrvList(optDic["servers"][NGAS_OPT_VAL])
    while (1):
        nextFileId = taskCtrl.getNextFileId()
        if (not nextFileId): thread.exit()
        nextFileId = nextFileId[:-1]
        info(1, "Next File ID: %s" % nextFileId)
        try:
            fileSize, fileObj = client.retrieveFileObj(nextFileId)
            sizeRemain = fileSize
            while (sizeRemain):
                if (sizeRemain < BLOCK_SIZE):
                    reqSize = sizeRemain
                else:
                    reqSize = BLOCK_SIZE
                buf = fileObj.read(reqSize)
                sizeRemain -= len(buf)
            taskCtrl.incBytesRecv(fileSize)
        except Exception, e:
            error("Error retrieving file with ID: %s - skipping. Error: %s" %\
                  (nextFileId, str(e)))
Example #13
0
def execute(optDic):
    """
    Carry out the tool execution.

    optDic:    Dictionary containing the options (dictionary).

    Returns:   Void.
    """
    info(4,"Entering execute() ...")
    if (optDic["help"][NGAS_OPT_VAL]):
        print correctUsage()
        sys.exit(0)

    # Submit the query.
    client = ngamsPClient.ngamsPClient()
    client.parseSrvList(optDic["SERVER"][NGAS_OPT_VAL])
    parameters = [["query", urllib.quote(optDic["QUERY"][NGAS_OPT_VAL])]]
    if (optDic["format"][NGAS_OPT_VAL]):
        parameters.append(["format", optDic["FORMAT"][NGAS_OPT_VAL]])
    if (optDic["reload"][NGAS_OPT_VAL]):
        parameters.append(["reload", "1"])
    startTime = time.time()
    stat = client.sendCmdGen("", -1, "QUERY", pars=parameters)
    stopTime = time.time()
    print "# Status: %s" % stat.getStatus()
    print "# Query:  %s" % optDic["QUERY"][NGAS_OPT_VAL]
    print "# Time:   %.6fs\n" % (stopTime - startTime)

    # Handle the response, dump the result to stdout in case of success.
    if (stat.getStatus() == NGAMS_SUCCESS):
        print stat.getData()
    else:
        msg = "Error occurred executing query: %s" % stat.getMessage()
        raise Exception, msg

    info(4,"Leaving execute()")
Example #14
0
def ngamsGLEAM_uimg_jobpi(srvObj, plugInPars, filename, fileId, fileVersion,
                          diskId):
    """
    srvObj:        Reference to NG/AMS Server Object (ngamsServer).

    plugInPars:    Parameters to take into account for the plug-in
                   execution (string).(e.g. scale_factor=4,threshold=1E-5)

    fileId:        File ID for file to test (string).

    filename:      Filename of (complete) (string).

    fileVersion:   Version of file to test (integer).

    Returns:       the return code of the compression plugin (integer).
    """
    """
    1. run the update_img_val.py to produce output at /tmp/out.fits
    2. update CRC and replace the current file
    3. clean up both outfilename
    """
    # 1. execute the update function
    outfilename = "{0}/{2}_{1}".format(tmp_path, fileId, fileVersion)
    cmd = "{0} {1} {2} {3}".format(py, py_exec, filename, outfilename)
    info(3, "IMUPDATE - Executing: " + cmd)
    re = execCmd(cmd)

    # 2. update CRC and replace the current file
    progress = 0
    try:
        info(3,
             'IMUPDATE - Executing: Getting crc from {0}'.format(outfilename))
        crc = getFileCRC(outfilename)
        info(3, "IMUPDATE - Executing: " + sql_crc)
        srvObj.getDb().query2(sql_crc, args=(crc, fileId, fileVersion, diskId))
        cmd_cp = "cp {0} {1}".format(outfilename, filename)
        info(3, "IMUPDATE - Executing: " + cmd_cp)
        execCmd(cmd_cp)
        return (0, 'OK')
    except Exception, ex:
        msg = "Updating CRC and replacing file {0} failed: {1}".format(
            outfilename, ex)
        error(msg)
        return (500, msg)
Example #15
0
def ingestExtFiles(fileListFile, notifEmail):
    """
    Ingest the list of files given in the file referred to.

    The files are listed as:

    <Disk ID> <File ID> [<File Version>]
    ...


    fileListFile:   Filename of file containing the file list (string).

    notifEmail:     List of email addresses to inform about the
                    execution of the discation procedure (string).

    Returns:        Void.
    """
    fileInfoList = parseFileList(fileListFile)

    # Open DB connection.
    server, db, user, password = ngasUtilsLib.getDbPars()
    dbCon = ngamsDb.ngamsDb(server, db, user, password, 0)

    # Find distinct Disk IDs.
    diskIdDic = {}
    for fileInfo in fileInfoList:
        diskIdDic[fileInfo[0]] = None
    info(1, "Disk IDs: " + str(diskIdDic.keys()))

    # Register disks referred to as external disks in the DB.
    dbDiskDic = {}
    for diskId in diskIdDic.keys():
        if (not dbDiskDic.has_key(diskId)):
            diskInfo = dbCon.getDiskInfoFromDiskId(diskId)
            if (diskInfo != []):
                diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\
                              unpackSqlResult(diskInfo)
            else:
                # Create a new entry for that disk in the DB.
                info(1,"Creating new entry for external disk with ID: " +\
                     str(diskId))
                diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\
                              setArchive("EXTERNAL").\
                              setDiskId(diskId).\
                              setLogicalName(diskId).\
                              setHostId("").\
                              setSlotId("").\
                              setMounted(0).\
                              setMountPoint("EXTERNAL").\
                              setNumberOfFiles(0).\
                              setAvailableMb(0).\
                              setBytesStored(0).\
                              setCompleted(1).\
                              setCompletionDateFromSecs(0).\
                              setType("EXTERNAL").\
                              setManufacturer("UNKNOWN").\
                              setInstallationDateFromSecs(time.time()).\
                              setChecksum(0).\
                              setTotalDiskWriteTime(0).\
                              setLastCheckFromSecs(0).\
                              setLastHostId("").\
                              setStorageSetId(diskId)
                diskInfoObj.write(dbCon)
            diskIdDic[diskId] = diskInfoObj

    # Loop over the files and register them in the DB.
    fileRegList = []
    sys.stdout.write("Registering files ...")
    fileCount = 0
    for fileInfo in fileInfoList:
        diskId = fileInfo[0]
        fileId = fileInfo[1]
        fileVer = fileInfo[2]
        fileInfoObj = ngamsFileInfo.ngamsFileInfo().\
                      setDiskId(diskId).\
                      setFilename(fileId).\
                      setFileId(fileId).\
                      setFileVersion(fileVer).\
                      setFormat("").\
                      setFileSize(0).\
                      setUncompressedFileSize(0).\
                      setCompression("").\
                      setIngestionDateFromSecs(0).\
                      setIgnore(0).\
                      setChecksum("").\
                      setChecksumPlugIn("").\
                      setFileStatus(NGAMS_FILE_STATUS_OK).\
                      setCreationDateFromSecs(0).\
                      setTag("EXTERNAL")
        fileInfoObj.write(getHostName(), dbCon, 0, 1)
        fileRegList.append((diskId, fileId, fileVer))
        time.sleep(0.050)
        fileCount += 1
        if ((fileCount % 10) == 0): sys.stdout.write(".")
    sys.stdout.write("\n")
    info(1, "Registered %d files" % fileCount)

    report = genReport(fileRegList)
    if (notifEmail):
        ngasUtilsLib.sendEmail("ngasRegisterExtFiles: FILE REGISTRATION " +\
                               "REPORT", notifEmail, report)
Example #16
0
    cmd:      Command (string).

    cmdPars:  List of parameters to send (list).

    Returns:  Tuple containing information about the test:

                (<Status (0|1)>, <XML Status>|<Info>|None>
    """
    info(4, "Entering sendCmd() ...")
    try:
        stat = ngamsPClient.ngamsPClient().setTimeOut(testParDic["TIMEOUT"]).\
               sendCmdGen(srvNode, srvPort, cmd, pars=cmdPars)
    except Exception, e:
        return (TEST_FAILURE, str(e))
    info(4, "Leaving sendCmd()")
    if (stat.getStatus() == NGAMS_SUCCESS):
        return (TEST_SUCCESS, stat.genXmlDoc())
    else:
        return (TEST_FAILURE, stat.genXmlDoc())


def execTest(testId, testParDic, dbmObj, srvNode, srvPort, testNode, testPort,
             cmd, cmdPars):
    """
    Execute a test +

    testId:     Test ID (string).

    testParDic: Dictionary with parameters for running the test
                (dictionary).
Example #17
0
def serverLoop(testParDic, serverList, ignoreSrvList, domainList, dbCon,
               dbmObj):
    """
    Loop over the contact servers and execute the tests for each node
    in the given domains, going through the contact node.

    testParDic:    Dictionary with parameters for running the test
                   (dictionary).

    serverList:    List of node/ports to contact for the tests (list).

    ignoreSrvList: List of servers to ignore (list).

    domainList:    List of domains to take into account (list).

    dbCon:         DB connection object (ngamsDb).

    dbmObj:        DBM object (ngamsDbm).

    Returns:       Number of problems encountered (integer).
    """
    info(4, "Entering serverLoop() ...")
    problemCount = 0

    # Get list of nodes.
    sqlQuery = "SELECT host_id, srv_port FROM ngas_hosts " +\
               "WHERE domain IN ('%s') " +\
               "AND (srv_state='ONLINE' OR srv_suspended=1)"
    sqlQuery = sqlQuery % str(testParDic["DOMAINS"]).strip()
    nodeList = dbCon.query(sqlQuery)
    for server in serverList:
        srvNode, srvPort = server.split(":")
        info(2, "Server: %s:%s" % (srvNode, str(srvPort)))
        for testNode, testPort in nodeList[0]:
            # Check if this server should be ignored.
            if (ngamsLib.elInList(ignoreSrvList, "%s:%s" %\
                                  (testNode, str(testPort)))):
                continue

            info(2, "  - Test node: %s/port: %s" % (testNode, testPort))
            for testCase in ["test_status_host_id", "test_retrieve_files"]:
                info(2, "    - Test: %s ..." % testCase)
                stat = loadInvokeTest(testCase, testParDic, dbCon, dbmObj,
                                      srvNode, srvPort, testNode, testPort,
                                      None)
                if (not stat):
                    info(2, "      - Result: SUCCESS")
                else:
                    info(2, "      - Result: FAILURE")
                problemCount += stat

    # Now, loop again over the failing nodes to cross-check.
    for n in range(3):
        problemCount = 0
        for nodeKey in dbmObj.keys():
            testRes = dbmObj.get(nodeKey)
            if (testRes[TEST_STAT] == TEST_FAILURE):
                testCase = testRes[TEST_ID]
                srvNode = testRes[SRV_HOST_ID]
                srvPort = testRes[SRV_PORT_NO]
                testNode = testRes[HOST_ID]
                testPort = testRes[PORT_NO]

                # Is this node online at all?
                onlineNodeList = nodeList[0]
                testNodePort = (testNode, testPort)
                if (not ngamsLib.elInList(onlineNodeList, testNodePort)):
                    continue

                # Ignore this node?
                testNodePort = "%s:%s" % (testNode, str(testPort))
                if (ngamsLib.elInList(ignoreSrvList, "%s" % testNodePort)):
                    continue

                msg = "Retrying test case: %s on node: %s:%s using server: "+\
                      "%s:%s"
                info(
                    2, msg %
                    (testCase, testNode, str(testPort), srvNode, str(srvPort)))
                stat = loadInvokeTest(testCase, testParDic, dbCon, dbmObj,
                                      srvNode, srvPort, testNode, testPort,
                                      None)
                if (stat): problemCount += 1
        # Make a small sleep if problems were found and we want to retry.
        if (problemCount and (n < 2)): time.sleep(10)

    info(4, "Leaving serverLoop()")
    return problemCount
Example #18
0
            elif (par.find("--SRCHOST") == 0):
                srcHost = sys.argv[idx].split("=")[-1]
            elif (par.find("--PATH") == 0):
                path = sys.argv[idx].split("=")[-1]
            elif (par.find("--OUTPUTDIR") == 0):
                outputDir = sys.argv[idx].split("=")[-1]
            elif (par == "--LIST"):
                list = 1
            else:
                raise Exception, "Unknown parameter: %s" % parOrg
            idx += 1
        except Exception, e:
            print "\nProblem executing the tool: %s\n" % str(e)
            print correctUsage()
            sys.exit(1)
    if ((not host) and (not port) or (not path)):
        print correctUsage()
        raise Exception, "Incorrect/missing command line parameter(s)!"
    if (not srcHost): srcHost = host
    try:
        if (not outputDir): outputDir = os.getcwd()
        outputDir = os.path.normpath("%s/%s" % (outputDir, srcHost))
        downloadFiles(host, port, srcHost, path, outputDir, list)
        if (not list):
            info(1, "Finished downloading specified path: %s" % path)
    except Exception, e:
        print "Problem encountered handling request:\n\n%s\n" % str(e)
        sys.exit(1)

# EOF
Example #19
0
                testParDic["DB"] = sys.argv[idx][(sys.argv[idx].find("=") +
                                                  1):]
            elif (par.find("--MOUNTPOINT") == 0):
                testParDic["MOUNTPOINT"] = sys.argv[idx].split("=")[-1]
            elif (par.find("--NOOFFILES") == 0):
                testParDic["NOOFFILES"] = int(sys.argv[idx].split("=")[-1])
            elif (par.find("--VERBOSE") == 0):
                setLogCond(0, "", 0, "", int(sys.argv[idx].split("=")[-1]))
            else:
                raise Exception, "Unknown parameter: %s" % parOrg
            idx += 1
        except Exception, e:
            print "\nProblem executing the tool: %s\n" % str(e)
            print correctUsage()
            sys.exit(1)
    try:
        if ((not testParDic["BASEFILE"]) or (not testParDic["MOUNTPOINT"])
                or (not testParDic["NOOFFILES"])):
            print correctUsage()
            raise Exception, "Incorrect/missing command line parameter(s)!"
        startTime = time.time()
        createVolume(testParDic)
        deltaTime = (time.time() - startTime)
        timePerFile = (deltaTime / testParDic["NOOFFILES"])
        info(1, "Time: %.3fs, %.3fs/file" % (deltaTime, timePerFile))
    except Exception, e:
        print "Problem encountered during execution:\n\n%s\n" % str(e)
        sys.exit(1)

# EOF
Example #20
0
def downloadFiles(host, port, srcHost, path, outputDir, list=0):
    """
    Download the files matching the given pattern via the host specified,
    from the host(s) indicated.

    The files downloaded will be stored in a directory structure of the
    form:

      <Output Dir>/<Source Host>/<File Path>

    I.e., the filepath of the file is preserved, and the source host name is
    also kept.

    host:       Name of host to be contacted (string).

    port:       Port number of NG/AMS Server (integer).

    srcHost:    List of hosts to be contacted (<Host 1>,<Host 2>,...).
                Can also be a single host, wildcards is allowed (string).

    path:       File pattern of file(s) to be retrieved. If this is a
                directory, the files in the directory will be retrieved
                (string).

    outputDir:  Output directory. If not given, the current working point
                will be chosen (string).

    list:       If set to 1 and the path given is a directory, rather than
                retrieving the file in the remote directory, the names of
                the remote files are listed on stdout (integer/0|1).

    Returns:    Void.
    """
    if (not list): commands.getstatusoutput("mkdir -p %s" % outputDir)
    reply, msg, hdrs, data = ngamsLib.httpGet(host,
                                              port,
                                              NGAMS_RETRIEVE_CMD,
                                              pars=[["internal", path],
                                                    ["host_id", srcHost]],
                                              timeOut=10)
    if (isNgasXmlStatusDoc(data)):
        stat = ngamsStatus.ngamsStatus().unpackXmlDoc(data, 1)
        if (stat.getStatus() == NGAMS_FAILURE):
            msg = "Problem handling request: %s" % stat.getMessage()
            raise Exception, msg
    else:
        stat = None
    if (data.find("<?xml version=") == -1):
        if (not list):
            storeFile(hdrs, data, outputDir)
        else:
            print path
        return

    # The reply is a file list. Retrieve the file in question.
    remDir = stat.getFileListList()[0].getComment().split(": ")[-1]
    remDir = os.path.normpath(remDir)
    outputDir = os.path.normpath("%s/%s" % (outputDir, remDir.split("/")[-1]))
    if (not list): commands.getstatusoutput("mkdir -p %s" % outputDir)
    client = ngamsPClient.ngamsPClient()
    for fileInfo in stat.getFileListList()[0].getFileInfoObjList():
        # If we should only list the contents and if the entry is a file,
        # we only display its information.
        if (list):
            print "%s %-8s %-8s %-10s %-16s %s" %\
                  (fileInfo.getPermissions(), fileInfo.getOwner(),
                   fileInfo.getGroup(), str(fileInfo.getFileSize()),
                   fileInfo.getModDate(), fileInfo.getFilename())
            # If the entry is a file and we should only list, we go to the
            # next item.
            if (fileInfo.getPermissions()[0] != "d"): continue

        # Handle/download the item.
        fn = fileInfo.getFilename()
        basename = os.path.basename(fn)
        trgFile = os.path.normpath("%s/%s" % (outputDir, basename))
        if (not list): info(1, "Requesting: %s:%s ..." % (srcHost, fn))
        reply2, msg2, hdrs2, data2 = \
                ngamsLib.httpGet(host, port, NGAMS_RETRIEVE_CMD,
                                 pars=[["internal", fn],
                                       ["host_id", srcHost]], timeOut=10)

        # If the remote object specified is a file, retrieve it, otherwise if a
        # directory, call this function recursively to retrieve the contents of
        # the sub-folder(s).
        if (isNgasXmlStatusDoc(data2)):
            stat2 = ngamsStatus.ngamsStatus().unpackXmlDoc(data2, 1)
            if (stat2.getStatus() == NGAMS_FAILURE):
                msg = "Problem handling request: %s" % stat2.getMessage()
                raise Exception, msg
            # It seems to be another File List, we retrieve the contents
            # of this recursively.
            if (not list): info(1, "Remote object found: %s" % fn)
            downloadFiles(host, port, srcHost, fn, outputDir, list)
        else:
            # Just retrieve the file.
            storeFile(hdrs2, data2, outputDir)

    return