コード例 #1
0
 def __init__(self, mygl, mynWaitMsec):
     threading.Thread.__init__(self, name="endall")
     self.gl = mygl
     self.nWaitMsec = mynWaitMsec
     self.llsFullOutput = list()
     NTRC.ntracef(2, "END", "exit init gl|%s| wait|%s|" 
                 % (self.gl, self.nWaitMsec))
コード例 #2
0
def fnDoOneJob(mytInstruction):
    ''' Execute a single job: 
        Do all lines.  
        Log results and convey to output queue.
    '''
    # Get my name and number for ident.
    sWhoami = mp.current_process().name
    NTRC.ntracef(3, "DO1J", "proc procname|%s|" % (sWhoami))
    nProc = fnsGetProcessNumber(sWhoami)

    # Unpack instruction command list and other items.
    lInstructions = mytInstruction.cmdlist
    (sLogfileDir, sLogfileName) = (mytInstruction.logdir
                                , mytInstruction.logname)
    qToUse = CWorkers.getOutputQueue()

    lResults = fnlsDoOneCmdList(lInstructions)
    # Send official results to the log file.
    fnWriteLogFile((lResults), sLogfileDir, sLogfileName)

    # If an output queue specified, pack up the answer and send it.
    if qToUse:
        # And send a copy of results to the specified output queue.
        lPrefix = [("BEGIN results from " + sWhoami)]
        lSuffix = [("ENDOF results from " + sWhoami)]
        lResultsToSee = ['\n'] + lPrefix + lResults + lSuffix + ['\n']
        tAnswers = tLinesOut(procname=sWhoami, listoflists=lResultsToSee)
        qToUse.put(tAnswers)
コード例 #3
0
    def cmBeforeAudit(self):
        '''
        Before each audit cycle, check to see if any servers
         have exceeded their lifetimes.
        '''
        for (sServerID, cServer) in (util.fnttSortIDDict(G.dID2Server)):
            fCurrentLife = cServer.mfGetMyCurrentLife()
            fFullLife = cServer.mfGetMyFullLife()
            fBirthday = cServer.mfGetMyBirthday()
            bServerAlive = not cServer.mbIsServerDead()
            bServerActive = cServer.bInUse

            # Log that we are examining this server,
            #  but note if it's already dead.
            sStatus = "inuse" if bServerActive else ""
            sStatus = sStatus if bServerAlive else "dead"
            lg.logInfo(
                "SHOCK ", "t|%6.0f| audit+end check svr|%s| "
                "life|%.0f|=|%.1f|yr %s" %
                (G.env.now, sServerID, fFullLife, fFullLife / 10000, sStatus))
            NTRC.ntracef(
                3, "SHOK", "proc t|%6.0f| check expir? svr|%s| "
                "svrdefaulthalflife|%s| born|%s| currlife|%s|" %
                (G.env.now, sServerID, G.fServerDefaultHalflife, fBirthday,
                 fCurrentLife))
            # Check to see if the server's lifetime has expired.
            bDeadAlready = CShock.cmbShouldServerDieNow(sServerID)

        return G.nDeadOldServers
コード例 #4
0
def fnlsDoOneCmdList(mylLines):
    """Input: list of instructions generated for this case; 
     multiprocessing queue through which to report results.
    
    Remove blanks, comments, etc., from the instructions.  Each line that
     is not blank or comment is a command to be executed.  Blanks and 
     comments are written directly into the output.

    Output: list of commands and their output, to be logged and 
     sent to the supplied queue.
    """

    # Process all command lines of the instruction list and collect results.  
    lResults = []               # list of strings
    for nLine, sLine in enumerate(mylLines):
        if fnbDoNotIgnoreLine(sLine):
            # Genuine line; execute and collect answer line(s).  
            tAnswer = fntDoOneCmdLine(sLine)
            (nRtn, nErr, lResult) = (tAnswer.callstatus
                                    , tAnswer.cmdstatus
                                    , tAnswer.ltext)
            lResults.extend(lResult)
            NTRC.ntracef(4, "DOCL", "proc DoOneList line|%s| "
                        "lResult|%s|" 
                        % (nLine, lResult))
        else:
            # Comment or blank line; just append to results.
            lResults.extend([("-"*len(fnsGetTimestamp()))
                            , (fnsGetTimestamp() + "  " + sLine)])
            NTRC.ntracef(4, "DOCL", "proc DoOneCase line|%s| "
                        "comment|%s|" 
                        % (nLine, sLine))
    return lResults
コード例 #5
0
def fntMatchValue(mysLine, mydVar):
    '''\
    Extract value from line according to valueregex for var.
     If no value found, supply suitably disappointing string.  
    Get the right word from the line.
     If asked for word zero, use the whole line.  
     Makes the extraction harder, but sometimes necessary.
    '''
    sWordnumber = mydVar["wordnumber"]
    nWordnumber = int(sWordnumber)
    lWords = mysLine.split()
    if nWordnumber == 0:
        sWord = mysLine
    elif nWordnumber <= len(lWords):
        sWord = lWords[nWordnumber - 1]
    else:
        sWord = "nowordhere_indexoutofrange"
    sValueregex = mydVar["valueregex"]
    sVarname = mydVar["varname"]
    oMatch = re.search(sValueregex, sWord)
    NTRC.tracef(
        5, "MCHV",
        "proc MatchValue matching word var|%s| word|%s| valueregex|%s| matchobj|%s|"
        % (sVarname, sWord, sValueregex, oMatch))
    if oMatch:
        # Word matches the valueregex.  Save the value.
        sValue = oMatch.group(1)
        NTRC.tracef(3, "MCHV",
                    "proc addvalue name|%s| val|%s|" % (sVarname, sValue))
    else:
        # If not found, at least supply something conspicuous for printing.
        sValue = "novaluefound"
    return (sVarname, sValue)
コード例 #6
0
def fnnProcessAllInstructions(myitInstructionIterator):
    ''' 
    Get the set of instructions that match the user's criteria for this batch,
     and run them one by one.
    Each instruction (run) is executed once for each random seed value.
    Count the number of runs, and don't exceed the user's limit, if any.
    If the execution reports a serious error, stop the loop.
    '''
    nRunNumber = 0
    maxcount = int(g.nTestLimit)
    # Is this a completely fake test run?  Replace templates.
    if g.sTestFib.startswith("Y"):
        g.lTemplates = g.lFibTemplates

    # Process each instruction in turn.
    for dRawInstruction in myitInstructionIterator:
        NTRC.ntracef(3, "MAIN",
                     "proc main raw instruction\n|%s|" % (dRawInstruction))
        dInstruction = fndMaybeEnhanceInstruction(dRawInstruction)
        NTRC.ntracef(3, "MAIN",
                     "proc main enhanced instruction\n|%s|" % (dInstruction))

        # Execute each instruction many times, once for each random seed value.
        nRunNumber += 1
        fnnProcessOneInstructionManyTimes(nRunNumber, dInstruction)

        # If user asked for a short test run today, maybe stop now.
        maxcount -= 1
        if int(g.nTestLimit) > 0 and maxcount <= 0: break

    # That's all, folks.  All instructions have been queued and will
    #  eventually be processed.
    # Send the shutdown messages to worker processes.
    g.cWorkersInst.Close()
    return nRunNumber
コード例 #7
0
    def mServerIsDead(self, mysServerID, mysCollID):
        '''\
        Auditor calls us: a server is dead, no longer 
         accepting documents.  Remove server from active list, 
         find a new server, populate it.  
        '''
        NTRC.ntracef(3, "CLI", "proc deadserver1 client|%s| place coll|%s| "
            "to|%d|servers" 
            % (self.ID, mysCollID, len(self.lServersToUse)))
        lg.logInfo("CLIENT", "server died cli|%s| removed svr|%s| coll|%s| " 
            % (self.ID, mysServerID, mysCollID))

        cColl = G.dID2Collection[mysCollID]
        cColl.lServerIDs.remove(mysServerID)
        nCollValue = cColl.nValue
        lServersForCollection = self.mSelectServersForCollection(nCollValue)
        # The distribution params have already limited the 
        # set of servers in the select-for-collection routine.
        # If there are servers available, pick one.  Otherwise, 
        #  create a new server that's just like an old one and use it.
        if lServersForCollection:
            sServerToUse = lServersForCollection.pop(0)
        else:
            sServerToUse = CServer.fnsInventNewServer()
        lg.logInfo("CLIENT", "client|%s| assign new server|%s| to replace|%s|" 
            % (self.ID, sServerToUse, mysServerID))
        nDocs = self.mPlaceCollectionOnServer(mysCollID, sServerToUse)
        lg.logInfo("CLIENT", "client|%s| provisioned new server|%s| "
            "collection|%s| ndocs|%s|" 
            % (self.ID, sServerToUse, mysCollID, nDocs))
        self.nServerReplacements += 1
        return sServerToUse
コード例 #8
0
 def msGentlyFormat(self, mysCmd, mydVals, myg, myCG):
     '''
     Like string.format() but does not raise exception if the string
      contains a name request for which the dictionary does not have 
      a value.  Leaves unfulfilled name requests in place.  
     Method: construct a dictionary that contains something for every
      name requested in the string.  The value is either a supplied 
      value from the caller or a placeholder for the name request.  
      Then use the now-defanged string.format() method.
     This is way harder than it ought to be, grumble.  
     '''
     # Make a dictionary from the names requested in the string
     #  that just replaces the request '{foo}' with itself.  
     sReNames = '(:?\{([^\}]+)\})+'
     oReNames = re.compile(sReNames)
     lNameTuples = oReNames.findall(mysCmd)
     NTRC.ntracef(3,"FMT","proc gently tuples|%s|" % (lNameTuples))
     lNames = [x[1] for x in lNameTuples]
     dNames = dict(zip(lNames, map(lambda s: "{"+s+"}", lNames)))
     # Pick up any specified values in the global object 
     #  and from CLI args.
     dNames.update(dict(vars(myCG)))
     dNames.update(dict(vars(myg)))
     # And then add values from the specific instructions.
     dNames.update(mydVals)
     NTRC.ntrace(3,"proc gently dnames|%s|" % (dNames))
     sOut = mysCmd.format(**dNames)
     return sOut
コード例 #9
0
def fntRunEverything(mygl, qInstr, fnbQEnd, nWaitMsec, nWaitHowMany):
    '''Start an async job for each case.  Limit number of concurrent jobs
    to the size of the ltJobs vector.  
    When a job completes, ship its output upline and remove it from 
    the active lists.  
    
    Two separate threads:
    - Wait for an empty slot; get an instruction, start an async job.
    - Wait for an active job to complete and remove it from lists.  
    '''
    # Fill the list of jobs with empties.
    for i in range(mygl.nParallel + 1): mygl.ltJobs.append(None)
    mygl.lockJobList = threading.Lock()
    mygl.lockPrint = threading.Lock()

    # Create and start new threads
    NTRC.ntracef(5, "RUN", "proc make thread instances")
    mygl.thrStart = CStartAllCases(mygl, mygl.nCoreTimer, mygl.nStuckLimit
                            , qInstr, fnbQEnd)
    mygl.thrEnd = CEndAllCases(mygl, mygl.nCoreTimer, )
    mygl.llsFullOutput = [["",""]]
    #mygl.thrStart.start()
    #mygl.thrEnd.start()
    
    # Wait until all jobs have started and finished.
    if (mygl.thrStart.is_alive() and mygl.thrStart.is_alive()):
        mygl.thrStart.join()     # Runs out of instructions.
        mygl.thrEnd.join()       # Runs out of finished jobs.  
    
    return tWaitStats(ncases=mygl.nCasesDone
                , slot=mygl.nWaitedForSlot
                , done=mygl.nWaitedForDone
                , inst=mygl.nWaitedForInstr)
コード例 #10
0
 def __init__(self, mygl, mynWaitMsec):
     threading.Thread.__init__(self, name="endall")
     self.gl = mygl
     self.nWaitMsec = mynWaitMsec
     self.llsFullOutput = list()
     NTRC.ntracef(2, "END", "exit init gl|%s| wait|%s|" 
                 % (self.gl, self.nWaitMsec))
コード例 #11
0
 def mSelectServersForCollection(self, mynCollValue):
     '''\
     Get list of servers at this quality level.
     
     Return a random permutation of the list of servers.
     Oops, not any more.  Just a list of usable ones.  
     '''
     # Get list of all servers at this quality level.
     # Value level translates to quality required and nr copies.
     (nQuality, nCopies) = G.dDistnParams[mynCollValue][0]
     lServersAtLevel = [ll[1] for ll in G.dQual2Servers[nQuality]]
     '''\
     For most questions, all servers are functionally 
      identical.  Just take the right number of them.  We used
      to take a random permutation of the list of servers and 
      choose from those, hence the name "Perm", but don't waste
      the effort any more.  
     NEW: return only servers that are not already in use and not broken.
     '''
     lPermChosenAlive = [svr for svr in lServersAtLevel 
                         if not G.dID2Server[svr].bDead]
     lPermChosenAvail = [svr for svr in lPermChosenAlive 
                         if not G.dID2Server[svr].bInUse]
     NTRC.ntracef(3, "CLI", "proc servers chosen level|%s| alive|%s| "
         "full|%s|" 
         % (lServersAtLevel, lPermChosenAlive, lPermChosenAvail))
     # Just make sure there are enough of them to meet the client's needs.
     if len(lPermChosenAlive) < nCopies:
         # Not enough servers available; someone will have to create one.
         lPermChosen = []
     else:
         lPermChosen = lPermChosenAvail[0:nCopies]
     return lPermChosen
コード例 #12
0
    def mDestroyCopy(self, mysCopyID):
        try:
            nCopyIndex = self.lCopyIDs.index(mysCopyID)
        except ValueError:
            NTRC.tracef(
                0, "SHLF",
                "BUGCHECK copyID not found for removal|%s|" % (mysCopyID))
            return False
        # Remove doc and copy from current lists.
        del self.lCopyIDs[nCopyIndex]
        del self.lDocIDs[nCopyIndex]
        # Tell the server that the copy is gone.
        cCopy = G.dID2Copy[mysCopyID]
        sDocID = cCopy.sDocID
        self.cServer.mDestroyCopy(mysCopyID, sDocID, self.ID)
        # And give back the space it occupied.
        self.bContig = False
        cDoc = G.dID2Document[sDocID]

        # BZZZT: DO NOT put this region back into use.  It has already
        # suffered an error once and caused a document to fail.
        #self.nFreeSpace += cDoc.nSize
        NTRC.tracef(
            3, "SHLF", "proc mDestroyCopy remove doc|%s| copy|%s| "
            "idx|%d| size|%d| from shelf|%s| remainingdocs|%d| free|%d|" %
            (cCopy.sDocID, mysCopyID, nCopyIndex, cDoc.nSize, self.ID,
             len(self.lCopyIDs), self.nFreeSpace))
        # And, at long last, destroy the Copy oject itself.
        del cCopy
        return self.ID + "-" + sDocID + "-" + mysCopyID
コード例 #13
0
 def mDestroyCopy(self,mysCopyID):
     try:
         nCopyIndex = self.lCopyIDs.index(mysCopyID)
     except ValueError:
         NTRC.tracef(0, "SHLF", "BUGCHECK copyID not found for removal|%s|" 
             % (mysCopyID))
         return False
     # Remove doc and copy from current lists.  
     del self.lCopyIDs[nCopyIndex]
     del self.lDocIDs[nCopyIndex]
     # Tell the server that the copy is gone.
     cCopy = G.dID2Copy[mysCopyID]
     sDocID = cCopy.sDocID
     self.cServer.mDestroyCopy(mysCopyID, sDocID, self.ID)
     # And give back the space it occupied.  
     self.bContig = False
     cDoc = G.dID2Document[sDocID]
     
     # BZZZT: DO NOT put this region back into use.  It has already 
     # suffered an error once and caused a document to fail.  
     #self.nFreeSpace += cDoc.nSize
     NTRC.tracef(3, "SHLF", "proc mDestroyCopy remove doc|%s| copy|%s| "
         "idx|%d| size|%d| from shelf|%s| remainingdocs|%d| free|%d|" 
         % (cCopy.sDocID, mysCopyID, nCopyIndex, cDoc.nSize, self.ID, 
         len(self.lCopyIDs), self.nFreeSpace))
     # And, at long last, destroy the Copy oject itself.
     del cCopy
     return self.ID + "-" + sDocID + "-" + mysCopyID
コード例 #14
0
    def mServerIsDead(self, mysServerID, mysCollID):
        '''\
        Auditor calls us: a server is dead, no longer 
         accepting documents.  Remove server from active list, 
         find a new server, populate it.  
        '''
        NTRC.ntracef(3, "CLI", "proc deadserver1 client|%s| place coll|%s| "
            "to|%d|servers" 
            % (self.ID, mysCollID, len(self.lServersToUse)))
        lg.logInfo("CLIENT", "server died cli|%s| removed svr|%s| coll|%s| " 
            % (self.ID, mysServerID, mysCollID))

        cColl = G.dID2Collection[mysCollID]
        cColl.lServerIDs.remove(mysServerID)
        nCollValue = cColl.nValue
        lServersForCollection = self.mSelectServersForCollection(nCollValue)
        # The distribution params have already limited the 
        # set of servers in the select-for-collection routine.
        # If there are servers available, pick one.  Otherwise, 
        #  create a new server that's just like an old one and use it.
        if lServersForCollection:
            sServerToUse = lServersForCollection.pop(0)
        else:
            sServerToUse = CServer.fnsInventNewServer()
        lg.logInfo("CLIENT", "client|%s| assign new server|%s| to replace|%s|" 
            % (self.ID, sServerToUse, mysServerID))
        nDocs = self.mPlaceCollectionOnServer(mysCollID, sServerToUse)
        lg.logInfo("CLIENT", "client|%s| provisioned new server|%s| "
            "collection|%s| ndocs|%s|" 
            % (self.ID, sServerToUse, mysCollID, nDocs))
        self.nServerReplacements += 1
        return sServerToUse
コード例 #15
0
    def cmBeforeAudit(self):
        '''
        Before each audit cycle, check to see if any servers
         have exceeded their lifetimes.
        '''
        for (sServerID, cServer) in (util.fnttSortIDDict(G.dID2Server)):
            fCurrentLife = cServer.mfGetMyCurrentLife()
            fFullLife = cServer.mfGetMyFullLife()
            fBirthday = cServer.mfGetMyBirthday()
            bServerAlive = not cServer.mbIsServerDead()
            bServerActive = cServer.bInUse

            # Log that we are examining this server, 
            #  but note if it's already dead.
            sStatus = "inuse" if bServerActive else ""
            sStatus = sStatus if bServerAlive else "dead"
            lg.logInfo("SHOCK ", "t|%6.0f| audit+end check svr|%s| "
                "life|%.0f|=|%.1f|yr %s" 
                % (G.env.now, sServerID, fFullLife, fFullLife/10000, 
                sStatus))
            NTRC.ntracef(3, "SHOK", "proc t|%6.0f| check expir? svr|%s| "
                "svrdefaulthalflife|%s| born|%s| currlife|%s|" 
                % (G.env.now, sServerID, G.fServerDefaultHalflife, 
                fBirthday, fCurrentLife))
            # Check to see if the server's lifetime has expired. 
            bDeadAlready = CShock.cmbShouldServerDieNow(sServerID)

        return G.nDeadOldServers
コード例 #16
0
 def msGentlyFormat(self, mysCmd, mydVals, myg, myCG):
     '''
     Like string.format() but does not raise exception if the string
      contains a name request for which the dictionary does not have 
      a value.  Leaves unfulfilled name requests in place.  
     Method: construct a dictionary that contains something for every
      name requested in the string.  The value is either a supplied 
      value from the caller or a placeholder for the name request.  
      Then use the now-defanged string.format() method.
     This is way harder than it ought to be, grumble.  
     '''
     # Make a dictionary from the names requested in the string
     #  that just replaces the request '{foo}' with itself.
     sReNames = '(:?\{([^\}]+)\})+'
     oReNames = re.compile(sReNames)
     lNameTuples = oReNames.findall(mysCmd)
     NTRC.ntracef(3, "FMT", "proc gently tuples|%s|" % (lNameTuples))
     lNames = [x[1] for x in lNameTuples]
     dNames = dict(zip(lNames, map(lambda s: "{" + s + "}", lNames)))
     # Pick up any specified values in the global object
     #  and from CLI args.
     dNames.update(dict(vars(myCG)))
     dNames.update(dict(vars(myg)))
     # And then add values from the specific instructions.
     dNames.update(mydVals)
     NTRC.ntrace(3, "proc gently dnames|%s|" % (dNames))
     sOut = mysCmd.format(**dNames)
     return sOut
コード例 #17
0
def fnnProcessAllInstructions(myitInstructionIterator):
    ''' 
    Get the set of instructions that match the user's criteria for this batch,
     and run them one by one.
    Each instruction (run) is executed once for each random seed value.
    Count the number of runs, and don't exceed the user's limit, if any.
    If the execution reports a serious error, stop the loop.
    '''
    nRunNumber = 0
    maxcount = int(g.nTestLimit)
    # Is this a completely fake test run?  Replace templates.
    if g.sTestFib.startswith("Y"):
        g.lTemplates = g.lFibTemplates

    # Process each instruction in turn.
    for dRawInstruction in myitInstructionIterator: 
        NTRC.ntracef(3,"MAIN","proc main raw instruction\n|%s|" 
            % (dRawInstruction))
        dInstruction = fndMaybeEnhanceInstruction(dRawInstruction)
        NTRC.ntracef(3,"MAIN","proc main enhanced instruction\n|%s|" 
            % (dInstruction))

        # Execute each instruction once for each random seed value.
        nRunNumber += 1
        lManyInstr = fnltProcessOneInstructionManyTimes(nRunNumber
                            , dInstruction)
        g.lGiantInstr.extend(lManyInstr)
        
        # If user asked for a short test run today, maybe stop now.
        maxcount -= 1
        if int(g.nTestLimit) > 0 and maxcount <= 0: break

    return nRunNumber
コード例 #18
0
    def __init__(self,size,mysClientID,mysCollectionID):
        self.ID = "D" + str(self.getID())
# BEWARE: if we have more than 10,000 docs, a fixed-length 
#  representation will have to change.  Bad idea; don't use it.
#  Change the sorting algorithm instead.
#        self.ID = "D" + "%04d"%(self.getID())
#  So, don't use it.
        G.dID2Document[self.ID] = self
        G.nDocLastID = self.ID
        self.nSize = size
        
        # Who owns this doc
        self.sClientID = mysClientID        # Doc owned by what client
        self.sCollID = mysCollectionID      # Doc lives in what collection
        NTRC.ntracef(3,"DOC","proc init client|%s| created doc|%s| size|%d|" % (self.sClientID,self.ID,self.nSize))
        
        # Where are copies of this doc stored
        self.lServerIDs = list()            # What servers currently have this doc
        self.lCopyIDs = list()              # What copy IDs are there of this doc
        self.setServerIDsAll = set([])      # What servers have ever had a copy
        
        # How has the doc fared in the storage wars
        self.bMajorityRepair = False        # True if ever repaired from majority of copies
        self.bMinorityRepair = False        # True if ever repaired from minority of copies
        self.bDocumentLost = False          # True if completely lost, all copies lost
        self.bDocumentOkay = True           # True if never repaired or lost
        self.nRepairsMajority = 0           # Number of repairs of doc from majority copies
        self.nRepairsMinority = 0           # Number of repairs of doc from minority copies
コード例 #19
0
    def __init__(self, size, mysClientID, mysCollectionID):
        self.ID = "D" + str(self.getID())
        # BEWARE: if we have more than 10,000 docs, a fixed-length
        #  representation will have to change.  Bad idea; don't use it.
        #  Change the sorting algorithm instead.
        #        self.ID = "D" + "%04d"%(self.getID())
        #  So, don't use it.
        G.dID2Document[self.ID] = self
        G.nDocLastID = self.ID
        self.nSize = size

        # Who owns this doc
        self.sClientID = mysClientID  # Doc owned by what client
        self.sCollID = mysCollectionID  # Doc lives in what collection
        NTRC.ntracef(
            3, "DOC", "proc init client|%s| created doc|%s| size|%d|" %
            (self.sClientID, self.ID, self.nSize))

        # Where are copies of this doc stored
        self.lServerIDs = list()  # What servers currently have this doc
        self.lCopyIDs = list()  # What copy IDs are there of this doc
        self.setServerIDsAll = set([])  # What servers have ever had a copy

        # How has the doc fared in the storage wars
        self.bMajorityRepair = False  # True if ever repaired from majority of copies
        self.bMinorityRepair = False  # True if ever repaired from minority of copies
        self.bDocumentLost = False  # True if completely lost, all copies lost
        self.bDocumentOkay = True  # True if never repaired or lost
        self.nRepairsMajority = 0  # Number of repairs of doc from majority copies
        self.nRepairsMinority = 0  # Number of repairs of doc from minority copies
コード例 #20
0
 def mEvaluateMe(self):
     '''\
     Return tuple of four bools stating doc status.
     How many copies do I have left (if any)?
     '''
     nCopiesLeft = len(
                     filter(
                         (lambda sServerID:
                             self.mTestOneServer(sServerID))
                         ,self.lServerIDs)
                      )
     # Are there any or enough copies left from which to repair the doc?
     nNumberOfServers = len(self.setServerIDsAll)
     nMajorityOfServers = (nNumberOfServers + 1) / 2
     # Include results from previous audits (if any).
     (bOkay, bMajority, bMinority, bLost) = (self.bDocumentOkay, self.bMajorityRepair,self.bMinorityRepair,self.bDocumentLost)
     NTRC.ntracef(3,"DOC","proc mEvaluateMe doc|%s| ncopies|%s| nservers|%s| okay|%s| majority|%s| minority|%s| lost|%s|" % (self.ID,nCopiesLeft,nNumberOfServers,bOkay,bMajority,bMinority,bLost))
     if nCopiesLeft > 0:
         # If there is a majority of copies remaining, 
         # then unambiguous repair is possible.
         if nCopiesLeft < nNumberOfServers and nCopiesLeft >= nMajorityOfServers:
             bMajority = True
             bOkay = False
         # Some copies left, but not enough for unambiguous repair.
         # Record that forensics are required for this doc repair. 
         elif nCopiesLeft < nMajorityOfServers:
             bMinority = True
             bOkay = False
     # There are no remaining copies of the doc, 
     # it cannot be repaired ever, oops.  Permanent loss.  
     else:
         bLost = True
         bOkay = False
     return (bOkay,bMajority,bMinority,bLost)
コード例 #21
0
def fnSendOneJobSlowly(myInstruction, myqJobs):
    ''' Queue this instruction as a job.
        If the queue size gets out of hand, wait for
         some jobs to be removed from it.0
     '''
    # If qsize > hi threshold, wait for it to come down.
    """ Boy, the walrus operator would really come in handy here.
        But that would restrict us to Python versions >= 3.8.  
    if (nQSize := myqJobs.qsize()) > g.nQThreshHi:
        NTRC.ntracef(3, "QTHR", "proc qsize over hi  |%s|" % (nQSize))
        while (nQSize := myqJobs.qsize()) > g.nQThreshLo:
            time.sleep(g.nQThreshSleep)
        NTRC.ntracef(3, "QTHR", "proc qsize under lo |%s|" % (nQSize))
    """
    nQSize = myqJobs.qsize()
    if nQSize > g.nQThreshHi:
        NTRC.ntracef(3, "QTHR", "proc qsize over hi  |%s|" % (nQSize))
        while True:
            time.sleep(g.nQThreshSleep)
            nQSize = myqJobs.qsize()
            if nQSize < g.nQThreshLo:
                break
        NTRC.ntracef(3, "QTHR", "proc qsize under lo |%s|" % (nQSize))
    # Okay, now queue the job.
    myqJobs.put(myInstruction)
    return nQSize
コード例 #22
0
 def fnbIsItDone(self, mysInstructionId):
     dIsItDone = {"sDoneId": mysInstructionId}
     dMaybeDone = self.oDoneCollection.find_one(dIsItDone)
     NTRC.ntracef(
         3, "DB", "proc check donelist id|%s| list|%s|" %
         (mysInstructionId, dMaybeDone))
     return isinstance(dMaybeDone, dict)  # None if not found.
コード例 #23
0
 def mSelectServersForCollection(self, mynCollValue):
     '''\
     Get list of servers at this quality level.
     
     Return a random permutation of the list of servers.
     Oops, not any more.  Just a list of usable ones.  
     '''
     # Get list of all servers at this quality level.
     # Value level translates to quality required and nr copies.
     (nQuality, nCopies) = G.dDistnParams[mynCollValue][0]
     lServersAtLevel = [ll[1] for ll in G.dQual2Servers[nQuality]]
     '''\
     For most questions, all servers are functionally 
      identical.  Just take the right number of them.  We used
      to take a random permutation of the list of servers and 
      choose from those, hence the name "Perm", but don't waste
      the effort any more.  
     NEW: return only servers that are not already in use and not broken.
     '''
     lPermChosenAlive = [svr for svr in lServersAtLevel 
                         if not G.dID2Server[svr].bDead]
     lPermChosenAvail = [svr for svr in lPermChosenAlive 
                         if not G.dID2Server[svr].bInUse]
     NTRC.ntracef(3, "CLI", "proc servers chosen level|%s| alive|%s| "
         "full|%s|" 
         % (lServersAtLevel, lPermChosenAlive, lPermChosenAvail))
     # Just make sure there are enough of them to meet the client's needs.
     if len(lPermChosenAlive) < nCopies:
         # Not enough servers available; someone will have to create one.
         lPermChosen = []
     else:
         lPermChosen = lPermChosenAvail[0:nCopies]
     return lPermChosen
コード例 #24
0
def fntMatchValue(mysLine,mydVar):
    '''\
    Extract value from line according to valueregex for var.
     If no value found, supply suitably disappointing string.  
    Get the right word from the line.
     If asked for word zero, use the whole line.  
     Makes the extraction harder, but sometimes necessary.
    '''
    sWordnumber = mydVar["wordnumber"]
    nWordnumber = int(sWordnumber)
    lWords = mysLine.split()
    if nWordnumber == 0:
        sWord = mysLine
    elif nWordnumber <= len(lWords):
        sWord = lWords[nWordnumber-1]
    else: 
        sWord = "nowordhere_indexoutofrange"
    sValueregex = mydVar["valueregex"]
    sVarname = mydVar["varname"]
    oMatch = re.search(sValueregex,sWord)
    NTRC.tracef(5,"MCHV","proc MatchValue matching word var|%s| word|%s| valueregex|%s| matchobj|%s|" % (sVarname,sWord,sValueregex,oMatch))
    if oMatch:
        # Word matches the valueregex.  Save the value.
        sValue = oMatch.group(1)
        NTRC.tracef(3,"MCHV","proc addvalue name|%s| val|%s|" % (sVarname,sValue))
    else:
        # If not found, at least supply something conspicuous for printing.
        sValue = "novaluefound"
    return (sVarname,sValue)
コード例 #25
0
def fntRunEverything(mygl, qInstr, fnbQEnd, nWaitMsec, nWaitHowMany):
    '''Start an async job for each case.  Limit number of concurrent jobs
    to the size of the ltJobs vector.  
    When a job completes, ship its output upline and remove it from 
    the active lists.  
    
    Two separate threads:
    - Wait for an empty slot; get an instruction, start an async job.
    - Wait for an active job to complete and remove it from lists.  
    '''
    # Fill the list of jobs with empties.
    for i in range(mygl.nParallel + 1): mygl.ltJobs.append(None)
    mygl.lockJobList = threading.Lock()
    mygl.lockPrint = threading.Lock()

    # Create and start new threads
    NTRC.ntracef(5, "RUN", "proc make thread instances")
    mygl.thrStart = CStartAllCases(mygl, mygl.nCoreTimer, mygl.nStuckLimit
                            , qInstr, fnbQEnd)
    mygl.thrEnd = CEndAllCases(mygl, mygl.nCoreTimer, )
    mygl.llsFullOutput = [["",""]]
    #mygl.thrStart.start()
    #mygl.thrEnd.start()
    
    # Wait until all jobs have started and finished.
    if (mygl.thrStart.is_alive() and mygl.thrStart.is_alive()):
        mygl.thrStart.join()     # Runs out of instructions.
        mygl.thrEnd.join()       # Runs out of finished jobs.  
    
    return tWaitStats(ncases=mygl.nCasesDone
                , slot=mygl.nWaitedForSlot
                , done=mygl.nWaitedForDone
                , inst=mygl.nWaitedForInstr)
コード例 #26
0
 def __init__(self, name, life):
     self.ID = name
     self.life = life
     self._timer = rt.CResettableTimer(G.env, life, shockcall, shockinter, self.ID)
     NTRC.ntrace(0, "proc shock.init before waitfor t|%s|" % G.env.now)
     self._timer.start()
     G.env.process(self.waitforshock())
     NTRC.ntrace(0, "proc shock.init after  waitfor t|%s|" % G.env.now)
コード例 #27
0
def fniPutFileToDb(mysFilename, mysSeparator, myoCollection):
    '''
    Open file to iterable, then use that to add file to db.
    '''
    # Store fresh, new records into the collection.
    with open(mysFilename, "r") as fhInfile:
        NTRC.ntrace(3, "File {} opened.".format(mysFilename))
    
        nRec = fnnPutIterToDb(fhInfile,mysSeparator,myoCollection)
    return nRec
コード例 #28
0
def main(mysDbName, mysCollectionName):
    oDb = fnoOpenDb(mysDbName)
    betterbezero = fniClearCollection(oDb, mysCollectionName)

    sFilename = fnsGetFilename()
    oCollection = oDb[mysCollectionName]
    nRecordCount = fniPutFileToDb(sFilename, " ", oCollection)
    dTmp = oCollection.find_one()
    NTRC.ntrace(0,"======\n{}\n======".format(dTmp))
    NTRC.ntrace(0,"nRecs stored|{}|".format(nRecordCount))
コード例 #29
0
def fndgGetSearchSpace(mysDir, mysTyp, mydUserRuleDict):
    '''
    Produce instruction stream from instruction files and user rules.
    '''
    dFullDict = fndReadAllInsFiles(mysDir, mysTyp)
    (dTrimmedDict,
     dOriginalDict) = fntProcessAllUserRules(mydUserRuleDict, dFullDict)
    dFilteredDict = fndFilterResults(dTrimmedDict)
    fnvTestResults(dFilteredDict, dFullDict)
    NTRC.ntracef(3, "SRCH",
                 "proc GetSearchSpace:FilteredDict|%s|" % (dFilteredDict))
    return fndgCombineResults(dFilteredDict)
コード例 #30
0
def fndgGetSearchSpace(mysDir, mysTyp, mydUserRuleDict):
    '''
    Produce instruction stream from instruction files and user rules.
    '''
    dFullDict = fndReadAllInsFiles(mysDir, mysTyp)
    (dTrimmedDict,dOriginalDict) = fntProcessAllUserRules(mydUserRuleDict, 
                                    dFullDict)
    dFilteredDict = fndFilterResults(dTrimmedDict)
    fnvTestResults(dFilteredDict, dFullDict)
    NTRC.ntracef(3, "SRCH", "proc GetSearchSpace:FilteredDict|%s|" 
        % (dFilteredDict))
    return fndgCombineResults(dFilteredDict)
コード例 #31
0
def fntDoesLineMatchThisVar(mysLine, mynLineNr, mysVarname):
    '''\
    Check line against lineregex of var.
    Return tuple (matchobject, line, varname).
    '''
    dVar = g.dVars[mysVarname]
    sLineregex = dVar["lineregex"]
    oMatch = re.search(sLineregex,mysLine)
    NTRC.tracef(5,"MTLN","proc MatchLine try regex|%s| var|%s| nr|%s| line|%s| match|%s|" % (sLineregex,mysVarname,mynLineNr,mysLine,oMatch))
    if oMatch:
        NTRC.tracef(3,"LINE","proc MatchLine found line|%s|=|%s| var|%s| regex|%s|" % (mynLineNr,mysLine,mysVarname,sLineregex))
    return (oMatch, mysLine, mysVarname)
コード例 #32
0
def fnvGetEnvironmentOverrides():
    # Allow user to override number of cores to use today.
    # Utility routine looks at HW and possible user envir override.
    g.nCores = brokergetcores.fnnGetResolvedCores()
    NTRC.ntracef(0, "MAIN", "proc ncores|%s|" % (g.nCores))
    # Allow user to override the polite interval to use today.
    try:
        g.nPoliteTimer = int(os.getenv("NPOLITE", CG.nPoliteTimer)) 
        NTRC.ntracef(0, "MAIN", "proc politetimer|%s|msec" % (g.nPoliteTimer))
    except (ValueError, TypeError):
        raise TypeError("Environment variable NPOLITE must be "
                        "an integer number of milliseconds.")
コード例 #33
0
def fnTimerInt(objTimer, xContext):
    '''\
    Server life-span timer was interrupted to reschedule it, 
    probably by a shock, and presumably to a shorter life.
    But the server is still alive.
    '''
    NTRC.trace(
        3, "interrupt %s delay %s called from %s at %s." %
        (xContext, objTimer.delay, objTimer, G.env.now))
    lg.logInfo(
        "SERVER", "interrupted t|%6.0f| context|%s| delay|%s|" %
        (G.env.now, xContext, objTimer.delay))
    return (objTimer, xContext)
コード例 #34
0
 def mMergeEvaluation(self,mybOkay,mybMajority,mybMinority,mybLost):
     '''\
     Carefully combine new doc info with old from audits, if any.
     
     E.g., finally okay only if was okay and still is okay; 
     finally lost if was lost or is now lost.  
     '''
     NTRC.ntracef(3,"DOC","proc merge in|%s|%s|%s|%s| with doc|%s|%s|%s|%s|" % (mybOkay,mybMajority,mybMinority,mybLost,self.bDocumentOkay,self.bMajorityRepair,self.bMinorityRepair,self.bDocumentLost))
     self.bDocumentOkay = self.bDocumentOkay and mybOkay
     self.bMajorityRepair = self.bMajorityRepair or mybMajority
     self.bMinorityRepair = self.bMinorityRepair or mybMinority
     self.bDocumentLost = self.bDocumentLost or mybLost
     return (self.bDocumentOkay,self.bMajorityRepair,self.bMinorityRepair,self.bDocumentLost)
コード例 #35
0
 def mGlitchHappensNow(self):
     """Start a glitch happening right now.
     May be invoked from outside a CLifetime instance as well as 
     from inside."""
     fNow = G.env.now
     NTRC.ntracef(
         3, "LIFE", "proc glitch wait expired t|%6.0f| "
         "for shelf|%s| freq|%d| life|%.3f| interval|%.3f|" %
         (fNow, self.sShelfID, self.nGlitchFreq, self.fShelfLife,
          self.fShelfInterval))
     self.mGlitchHappens(fNow)
     lg.logInfo(
         "LIFETIME", "glitchnow t|%6.0f| for shelf|%s| active|%s|" %
         (fNow, self.sShelfID, self.bGlitchActive))
コード例 #36
0
 def __init__(self, mygl 
             , mynWaitMsec, mynWaitHowMany
             , myqInstructions, myfnbEnd
             ):
     threading.Thread.__init__(self, name="startall")
     self.gl = mygl
     self.nWaitMsec = mynWaitMsec
     self.nWaitHowMany = mynWaitHowMany
     self.nCounter = itertools.count(1)
     self.nProcess = 0
     self.qInstructions = myqInstructions
     self.fnbEnd = myfnbEnd
     NTRC.ntracef(2, "STRT", "exit init gl|%s| instrs|%s|" 
             % (self.gl, self.qInstructions))
コード例 #37
0
def main():
    NTRC.ntrace(0, "Begin.")
    # Get args from CLI and put them into the global data
    dCliDict = fndCliParse("")
    # Carefully insert any new CLI values into the Global object.
    dCliDictClean = {k: v for k, v in dCliDict.items() if v is not None}
    g.__dict__.update(dCliDictClean)

    # Use naked Mongo functions not suitable for searchdatabasemongo library.
    # Since MongoDB is a system-wide singleton resource, there is no need
    #  to get any name arguments for this command.
    client = pymongo.MongoClient()
    client.drop_database(g.sDatabaseName)
    NTRC.ntrace(0, "End.")
コード例 #38
0
def fnldParseInput(mysFilename):
    ''' Return tuple containing
        - the output template string, 
        - a list, one item per line, of dicts of column args from the 
          csv that contain instructions for getting variable values
          from lines.  
        Beware duck-type integers that become strings.

        Format of csv lines:        
        varname,regex to find line,split word number,regex to strip out value

        instruction file format:

        ##becomes comment in output
        ###send out this string as header for the output, no hashes
        =outputformat
        format string
        =variables
        varname,lineregex,wordnumber,valueregex (header)
        (lines of csv data)

    '''
    dParams = dict()
    with open(mysFilename,"rb") as fhInfile:
        # Remove comments.  
        lLines = filter( lambda sLine:                          \
                        not re.match("^ *#[^#]",sLine)          \
                        and not re.match("^ *$",sLine.rstrip()) \
                        , fhInfile )

        # Get the output template.  It may be longer than one line.  
        lTemplate = fnlLinesInRange(lLines,"^=template","^=variables")
        lTemplate = map( lambda sLine: sLine.rstrip().replace("###","").replace("##","#"), lTemplate )
        NTRC.tracef(3,"INPT","proc ParseInput template|%s|" % (lTemplate))

        # Fix the separator in the template according to the user spec.
        lAllTemplateNames = [lTemplateLine.split() for lTemplateLine in lTemplate]
        lNewTemplate = [g.sSeparator.join(lTemplateNamesOneLine) \
            for lTemplateNamesOneLine in lAllTemplateNames]

        # Now get the CSV args into a dictionary of dictionaries.
        lVarLines = fnlLinesInRange(lLines,"^=variables","^=thiswillnotbefound")
        lRowDicts = csv.DictReader(lVarLines)
        NTRC.tracef(5,"INPT","proc ParseInput lRowDicts all|%s|" % (lRowDicts))
        
        dParams = dict( map( lambda dRowDict:   \
            (dRowDict["varname"],dRowDict)      \
            , lRowDicts ))

    return (lNewTemplate,dParams)
コード例 #39
0
 def __init__(self, mygl 
             , mynWaitMsec, mynWaitHowMany
             , myqInstructions, myfnbEnd
             ):
     threading.Thread.__init__(self, name="startall")
     self.gl = mygl
     self.nWaitMsec = mynWaitMsec
     self.nWaitHowMany = mynWaitHowMany
     self.nCounter = itertools.count(1)
     self.nProcess = 0
     self.qInstructions = myqInstructions
     self.fnbEnd = myfnbEnd
     NTRC.ntracef(2, "STRT", "exit init gl|%s| instrs|%s|" 
             % (self.gl, self.qInstructions))
コード例 #40
0
def fnvGetEnvironmentOverrides():
    # Allow user to override number of cores to use today.
    # Utility routine looks at HW and possible user envir override.
    g.nCores = brokergetcores.fnnGetResolvedCores()
    NTRC.ntracef(0, "MAIN", "proc ncores|%s|" % (g.nCores))
    g.nParallel = g.nCores  # Sorry for the name change.
    # Allow user to override the polite interval to use today.
    try:
        g.nPoliteTimer = int(os.getenv("NPOLITE", CG.nPoliteTimer))
        g.nCoreTimer = g.nPoliteTimer  # Sorry for the name change.
        NTRC.ntracef(0, "MAIN", "proc politetimer|%s|msec" % (g.nPoliteTimer))
    except (ValueError, TypeError):
        raise TypeError("Environment variable NPOLITE must be "
                        "an integer number of milliseconds.")
コード例 #41
0
def fnnCalcDocSize(mynLevel):
    lPercents = G.dDocParams[mynLevel]
    nPctRandom = makeunif(0,100)
    nPctCum = 0
    for lTriple in lPercents:
        (nPercent, nMean, nSdev) = lTriple
        nPctCum += nPercent
        if nPctRandom <= nPctCum:
            nDocSize = int(makennnorm(nMean, nSdev))
            NTRC.ntracef(3,"DOC","proc CalcDocSize rand|%s| cum|%s| pct|%s| "
            "mean|%s| sd|%s| siz|%s|" 
            % (nPctRandom,nPctCum,nPercent,nMean,nSdev,nDocSize))
            break
    return nDocSize
コード例 #42
0
def main():
    NTRC.ntrace(0,"Begin.")
    # Get args from CLI and put them into the global data
    dCliDict = fndCliParse("")
    # Carefully insert any new CLI values into the Global object.
    dCliDictClean = {k:v for k,v in dCliDict.items() if v is not None}
    g.__dict__.update(dCliDictClean)

    # Use naked Mongo functions not suitable for searchdatabasemongo library. 
    # Since MongoDB is a system-wide singleton resource, there is no need 
    #  to get any name arguments for this command.   
    client = pymongo.MongoClient()
    client.drop_database(g.sDatabaseName)
    NTRC.ntrace(0,"End.")
コード例 #43
0
 def mDestroyCopy(self, mysCopyID, mysDocID, mysShelfID):
     ''' Oops, a doc died, maybe just one or maybe the whole shelf.
     '''
     NTRC.tracef(
         3, "SERV", "proc mDestroyCopy remove copy|%s| doc|%s| "
         "from shelf|%s|" % (mysCopyID, mysDocID, mysShelfID))
     # Inform the client that the copy is gonzo.
     cClient = G.dID2Client[self.dDocIDs[mysDocID]]
     cClient.mDestroyCopy(mysDocID, self.ID, mysCopyID)
     # Clear out local traces of the doc and copy.
     self.lDocIDs.remove(mysDocID)
     del self.dDocIDs[mysDocID]
     # The Shelf will nuke the copy, because it created it.
     return self.ID + "-" + mysDocID
コード例 #44
0
 def m_CreateWorkers(self):
     ''' Create worker pool and start them all. 
         The number of workers was specified in the class construction.
     '''
     lProcs = []
     for iProcessNum in range(self.nParallel):
         proc = mp.Process(target=doManyJobs,
                           args=(self.qJobs, ),
                           name=f'CWorker-{iProcessNum+1}')
         lProcs.append(proc)
         proc.start()
     NTRC.ntrace(3, "proc worker|%s| started on q|%s|" % (proc, self.qJobs))
     self.lprocWorkers = lProcs  # Save a list of all workers.
     return lProcs
コード例 #45
0
def doManyJobs(myqJobs):
    ''' This is the guy who gets called as a job worker
        Read a job from input queue.
        If it is a real instruction, do it.
        If it is an end code, exit.
        '''
    while True:
        tInstructionJob = myqJobs.get()
        sWhoami = mp.current_process().name
        NTRC.ntracef(3, "DOMJ", "proc DoManyJobs|%s| qget|%s|" % (sWhoami, tInstructionJob,))
        if tInstructionJob.cmdlist:
            result = fnDoOneJob(tInstructionJob)
        else:
            sys.exit(0)
コード例 #46
0
 def starter4e(env, event):
     NTRC.trace(3,"4e.starter4e: before callstart at %s." % env.now)
     event.start()
     NTRC.trace(3,"4e.starter4e: before yieldtimeout at %s." % env.now)
     yield env.timeout(3)
     NTRC.trace(3,"4e.starter4e: after yieldtimeout at %s." % env.now)
     event.reset()
     NTRC.trace(3,"4e.starter4e: after callreset at %s." % env.now)
コード例 #47
0
 def mDestroyShelf(self):
     ''' Nuke all the copies on the shelf.  
         Can't delete the CShelf object, however.
     '''
     NTRC.ntracef(
         3, "SHLF", "proc mDestroyShelf1 shelf|%s| "
         "has ncopies|%s|" % (self.ID, len(self.lCopyIDs)))
     lg.logInfo(
         "SHELF ", "t|%6.0f| destroy shelf|%s| "
         "of svr|%s| ncopies|%s|" %
         (G.env.now, self.ID, self.sServerID, len(self.lCopyIDs)))
     lAllCopyIDs = self.lCopyIDs[:]  # DANGER: list modified inside loop,
     #  requires deepcopy.
     for sCopyID in lAllCopyIDs:
         self.mDestroyCopy(sCopyID)
コード例 #48
0
def fnnCalcDocSize(mynLevel):
    lPercents = G.dDocParams[mynLevel]
    nPctRandom = makeunif(0, 100)
    nPctCum = 0
    for lTriple in lPercents:
        (nPercent, nMean, nSdev) = lTriple
        nPctCum += nPercent
        if nPctRandom <= nPctCum:
            nDocSize = int(makennnorm(nMean, nSdev))
            NTRC.ntracef(
                3, "DOC", "proc CalcDocSize rand|%s| cum|%s| pct|%s| "
                "mean|%s| sd|%s| siz|%s|" %
                (nPctRandom, nPctCum, nPercent, nMean, nSdev, nDocSize))
            break
    return nDocSize
コード例 #49
0
 def mDestroyShelf(self):
     ''' Nuke all the copies on the shelf.  
         Can't delete the CShelf object, however.
     '''
     NTRC.ntracef(3, "SHLF", "proc mDestroyShelf1 shelf|%s| "
         "has ncopies|%s|" 
         % (self.ID, len(self.lCopyIDs)))
     lg.logInfo("SHELF ", "t|%6.0f| destroy shelf|%s| "
         "of svr|%s| ncopies|%s|" 
         % (G.env.now, self.ID, self.sServerID, 
         len(self.lCopyIDs)))
     lAllCopyIDs = self.lCopyIDs[:]  # DANGER: list modified inside loop, 
                                     #  requires deepcopy.
     for sCopyID in lAllCopyIDs:
             self.mDestroyCopy(sCopyID)
コード例 #50
0
 def mInjectError(self, mynReduction, mynDecayHalflife, mynGlitchMaxlife):
     '''\
     When a glitch occurs, decrease lifetime by some amount, percentage.
     The decrease decays exponentially at some rate until negligible.  
     '''
     self.nReductionPercentage = mynReduction
     self.fDecayHalflife = float(mynDecayHalflife)
     self.fDecayRate = self.fLn2 / self.fDecayHalflife
     self.fMaxlife = float(mynGlitchMaxlife)
     NTRC.tracef(
         3, "LIFE", "proc inject reduct|%s| decayhalflife|%s| "
         "decayrate|%s| maxlife|%s|" %
         (self.nReductionPercentage, self.fDecayHalflife, self.fDecayRate,
          self.fMaxlife))
     return self.fDecayRate
コード例 #51
0
def defaultReceiveOutput(myqOutput):
    while True:
        tAnswers = myqOutput.get()
        sWhoami = mp.current_process().name
        NTRC.ntracef(
            3, "RCVO",
            "proc DefRcvOut|%s| got output |%s|" % (sWhoami, repr(tAnswers)))
        lOutput = tAnswers.listoflists
        if lOutput:
            # Print it all on stdout.
            for sLine in lOutput:
                print(sLine)
            print("--------------")
        else:
            sys.exit(0)
コード例 #52
0
def mainsim_setup_post():
    #   C O L L E C T   D A T A 
    sFamilyDir = bottle.request.forms.get("sFamilyDir")
    sSpecificDir = bottle.request.forms.get("sSpecificDir")
    bClearDirs = bottle.request.forms.get("bClearDirs")
    bClearDone = bottle.request.forms.get("bClearDone")
    sAction = bottle.request.forms.get("submit")
    sOK = bottle.request.POST.ok
    sCancel = bottle.request.POST.cancel

    msg = "mainsim_setup_post: done"

    #   F O R M   D I C T I O N A R Y   O F   S U B S T I T U T I O N S 
    # Make a dictionary to use to substitute params into CLI command.
    dVals = dict(sFamilyDir=sFamilyDir
                ,sSpecificDir=sSpecificDir
                ,bClearDirs=("Yes" if bClearDirs else "No")
                ,bClearDone=("Yes" if bClearDone else "No")
                ,sOK=sOK
                ,sCancel=sCancel
                ,sAction=("DONE" if sOK else "CANCELLED")
                ,msg=msg
                )
    NTRC.ntrace(3,"proc first dict|%s|" % (dVals))

    if sOK:
        # If instructed to clear area, do that first.
        if bClearDirs:
            sClearDirsCmd = cCmd.mMakeCmd(sCmdClearDirs, dVals)
            lCmdOut = cCmd.mDoCmdLst(sClearDirsCmd)
            dVals["sResultClearDirs"] = fnsMakeReadable(sClearDirsCmd, lCmdOut)
        
        # Use standard script to setup output dirs.
        sSetupDirsCmd = cCmd.mMakeCmd(sCmdSetupDirs, dVals)
        lCmdOut = cCmd.mDoCmdLst(sSetupDirsCmd)
        dVals["sResultSetupDirs"] = fnsMakeReadable(sSetupDirsCmd, lCmdOut)

        # Use standard script to clear done records.
        if bClearDone:
            sClearDoneCmd = cCmd.mMakeCmd(sCmdClearDone, dVals)
            lCmdOut = cCmd.mDoCmdLst(sClearDoneCmd)
            dVals["sResultClearDone"] = fnsMakeReadable(sClearDoneCmd, lCmdOut)

    # Tell user what we did.
    lVals = ["k:|%s| v:|%s|" % (k, v) for (k, v) in sorted(dVals.items())]
    sVals = "\n<br/>".join(lVals)
    sOut = sVals
    return dVals
コード例 #53
0
def fndgCombineResults(mydInstructions):
    '''
    Expand the cross product of remaining instruction values.
    '''
    lKeyNames = [k for k in mydInstructions.keys()]
    for lInstruction in itertools.product(*[mydInstructions[sKey] 
                                        for sKey in lKeyNames]):
        dInstruction = dict(zip(lKeyNames, lInstruction)) 
        # Add unique id, as Mongo does, so we can find jobs already done.
###        dInstruction["_id"] = hashlib.sha1(str(dInstruction)).hexdigest()
        dInstruction["_id"] = hashlib.sha1(str(dInstruction).encode('ascii')).hexdigest()
        NTRC.ntracef(3, "SRCH", "proc CombineResults:dInstruction|%s|" 
            % (dInstruction))
        yield dInstruction
    
    '''
コード例 #54
0
def main():
    NTRC.ntrace(0,"Begin.")
    # Get args from CLI and put them into the global data
    dCliDict = fndCliParse("")
    # Carefully insert any new CLI values into the Global object.
    dCliDictClean = {k:v for k,v in dCliDict.items() if v is not None}
    g.__dict__.update(dCliDictClean)

    # Since we're deleting an arbitrary collection from the db, 
    #  it doesn't matter if we pretend that it is a specific one
    #  with a different name today.  
    g.mdb = searchdatabasemongo.CSearchDatabase(g.sDatabaseName, 
                    g.sCollectionName, g.sCollectionName)
    g.mdb.fnvDeleteProgressCollection()

    NTRC.ntrace(0,"Cleared collection|{1}| in database|{0}|".format(g.sDatabaseName,g.sCollectionName))
コード例 #55
0
def main():
    NTRC.ntrace(0, "Begin.")
    # Get args from CLI and put them into the global data
    dCliDict = fndCliParse("")
    # Carefully insert any new CLI values into the Global object.
    dCliDictClean = {k: v for k, v in dCliDict.items() if v is not None}
    g.__dict__.update(dCliDictClean)

    # Since we're deleting an arbitrary collection from the db,
    #  it doesn't matter if we pretend that it is a specific one
    #  with a different name today.
    g.mdb = searchdatabasemongo.CSearchDatabase(g.sDatabaseName)
    lNames = g.mdb.fnlGetCollections()
    for sName in lNames:
        print sName
    NTRC.ntrace(0, "End.")
コード例 #56
0
 def waitforshock(self):
     NTRC.ntrace(0, "proc shock.waitfor before yield t|%s|" % G.env.now)
     yield self._timer.event
     NTRC.ntrace(0, "proc shock.waitfor after  yield t|%s|" % G.env.now)
     
     NTRC.ntrace(0, "proc shock.waitfor reset the server timer here!")
     G.sa.timer.stop()
     G.sa.timer.setdelay(33333).start()
     NTRC.ntrace(0, "proc shock.waitfor done reset server timer")
コード例 #57
0
 def starter4f(env, event):
     NTRC.trace(3,"4f.starter4f: before callstart at %s." % env.now)
     event.start()
     NTRC.trace(3,"4f.starter4f: before yieldtimeout at %s." % env.now)
     yield env.timeout(5)
     NTRC.trace(3,"4f.starter4f: after yieldtimeout at %s." % env.now)
     event.stop()
     event.start()
     NTRC.trace(3,"4f.starter4f: after stop/start at %s." % env.now)
コード例 #58
0
 def cmbShouldServerDieNow(self, mysServerID):
     ''' 
     If the server's (possibly reduced) lifetime has expired, 
      kill it rather than restoring it to a full life.
     '''
     cServer = G.dID2Server[mysServerID]
     fCurrentLife = cServer.mfGetMyCurrentLife()
     fFullLife = cServer.mfGetMyFullLife()
     fBirthday = cServer.mfGetMyBirthday()
     bServerAlive = not cServer.mbIsServerDead()
     if (G.fServerDefaultHalflife > 0
         and fCurrentLife > 0
         and fFullLife <= G.env.now
         and bServerAlive
         ):
         # Server has overstayed its welcome.  Kill it.  
         sInUse = "currently in use" if cServer.mbIsServerInUse() else ""
         sShockVictim = "shock victim" if cServer.mbIsServerInShock() else ""
         lg.logInfo("SHOCK ", "t|%6.0f| kill svr|%s| "
             "born|%.0f| life|%.0f|=|%.1f|yr "
             "expired %s %s" 
             % (G.env.now, mysServerID, fBirthday, 
             fCurrentLife, fCurrentLife/10000, 
             sInUse, sShockVictim))
         NTRC.ntracef(3, "SHOK", "proc t|%6.0f| expired svr|%s| "
             "svrdefaulthalflife|%s| born|%.0f| currlife|%.0f|" 
             % (G.env.now, mysServerID, G.fServerDefaultHalflife, 
             fBirthday, fCurrentLife))
         result = cServer.mKillServer()
         G.nDeadOldServers += 1
         bResult = True
         # Now check to see if the server died because of the shock.
         #  Is the current life less than the original life?
         # Philosophical question: if the shock type 2 caused your new, 
         #  recalculated life to be longer than your original life, 
         #  can your death reasonably be attributed to the shock?
         #  Answer = no, because without the shock you would have
         #  died even earlier.  Tricky, though.  
         fOriginalLife = cServer.mfGetMyOriginalLife()
         if fCurrentLife < fOriginalLife:
             G.nDeathsDueToShock += 1
             G.lDeathsDueToShock.append(mysServerID)
     else:
         bResult = False
     return bResult
コード例 #59
0
 def mTestClient(self):
     '''\
     Return list, maybe empty, of all documents missing from 
      this client.  All collections appended together.
     '''
     lDeadDocIDs = list()
     for sCollID in self.lCollectionIDs:
         cColl = G.dID2Collection[sCollID]
         lResult = cColl.mTestCollection()
         NTRC.ntracef(3, "CLI", "proc TestClient1 client|%s| "
             "tests coll|%s| result|%s|" 
             % (self.ID, sCollID, lResult))
         if len(lResult) > 0:
             lDeadDocIDs.extend(lResult)
             NTRC.ntracef(3, "CLI", "proc TestClient2 client |%s| "
                 "coll|%s| lost docs|%s|" 
                 % (self.ID, sCollID, lResult))
     return lDeadDocIDs
コード例 #60
0
def makeServers(mydServers):
    for sServerName in mydServers:
        (nServerQual,nShelfSize) = mydServers[sServerName][0]
        cServer = server.CServer(sServerName,nServerQual,nShelfSize)
        sServerID = cServer.ID
        G.lAllServers.append(cServer)
        fCurrentLife = cServer.mfGetMyCurrentLife()
        lg.logInfo("MAIN","created server|%s| quality|%s| shelfsize|%s|TB "
            "name|%s| life|%.0f|" 
            % (sServerID, nServerQual, nShelfSize, sServerName, fCurrentLife))
        # Invert the server list so that clients can look up 
        # all the servers that satisfy a quality criterion.  
        if nServerQual in G.dQual2Servers:
            G.dQual2Servers[nServerQual].append([sServerName,sServerID])
        else:
            G.dQual2Servers[nServerQual] = [[sServerName,sServerID]]
        NTRC.ntracef(5,"SVRS","proc makeServers dQual2Servers qual|%s| servers|%s|" % (nServerQual,G.dQual2Servers[nServerQual]))
    return G.dQual2Servers