def transferLogFiles(): """ @summary : Log files will not be tansferred if local machine is not designed to be a pickling machine. If log files are to be transferred, they will be straight from the source." """ paths = StatsPaths() paths.setPaths() parameters = StatsConfigParameters() machineParameters = MachineConfigParameters() machineParameters.getParametersFromMachineConfigurationFile() parameters.getAllParameters() individualSourceMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.sourceMachinesTags ) individualPicklingMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.picklingMachines ) for sourceMachine,picklingMachine in map( None, individualSourceMachines, individualPicklingMachines ) : if picklingMachine == LOCAL_MACHINE :#pickling to be done here userName = machineParameters.getUserNameForMachine(sourceMachine) remoteLogPath = paths.getPXPathFromMachine( paths.PXLOG, sourceMachine, userName ) print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( userName , sourceMachine,remoteLogPath , paths.STATSLOGS, sourceMachine ) output = commands.getoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( userName , sourceMachine, remoteLogPath, paths.STATSLOGS, sourceMachine ) ) print output
def transferLogFiles(): """ @summary : Log files will not be tansferred if local machine is not designed to be a pickling machine. If log files are to be transferred, they will be straight from the source." """ paths = StatsPaths() paths.setPaths() parameters = StatsConfigParameters() machineParameters = MachineConfigParameters() machineParameters.getParametersFromMachineConfigurationFile() parameters.getAllParameters() individualSourceMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.sourceMachinesTags) individualPicklingMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.picklingMachines) for sourceMachine, picklingMachine in map(None, individualSourceMachines, individualPicklingMachines): if picklingMachine == LOCAL_MACHINE: #pickling to be done here userName = machineParameters.getUserNameForMachine(sourceMachine) remoteLogPath = paths.getPXPathFromMachine(paths.PXLOG, sourceMachine, userName) print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " % ( userName, sourceMachine, remoteLogPath, paths.STATSLOGS, sourceMachine) output = commands.getoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " % (userName, sourceMachine, remoteLogPath, paths.STATSLOGS, sourceMachine)) print output
def updatePickles( parameters, machineParameters, currentTimeInIsoFormat ): """ @summary : Updates the pickle files for all the specified log machines so that they are available for graphic production. @note : Pickling is to be done on specified pickling machines. All the pickle files that are produced on remote machines will be downloaded on the local machine. @param parameters: StatsConfigParameters instance containing the parameters found in the config file. @param machineParameters: MachineConfigParameters instance containing the parameters found in the config file. @param currentTimeInIsoFormat : Time at which this program was originally called. """ nbChildProcess = 0 paths = StatsPaths() paths.setPaths() for tag in parameters.sourceMachinesTags: pid = os.fork() if pid == 0: #if child sourceMachines = machineParameters.getMachinesAssociatedWith(tag) for i in range( len( sourceMachines ) ): picklingMachine = parameters.detailedParameters.picklingMachines[tag][i] # If pickling and source machines differ, download log files frm source to pickling machine. if sourceMachines[i] != picklingMachine: if parameters.detailedParameters.picklingMachines[tag][i] != LOCAL_MACHINE :#pickling to be done elsewhere for j in range(3):#do 3 times in case of currently turning log files. remotePxLogPath = paths.getPXPathFromMachine(paths.PXLOG, sourceMachines[i], machineParameters.getUserNameForMachine( sourceMachines[i])) output = commands.getoutput( "ssh %s@%s 'rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/' " %( machineParameters.getUserNameForMachine( picklingMachine), picklingMachine,machineParameters.getUserNameForMachine( sourceMachines[i] ) , sourceMachines[i] , remotePxLogPath, paths.STATSLOGS, sourceMachines[i] ) ) print "ssh %s@%s 'rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/' " %( machineParameters.getUserNameForMachine( picklingMachine), picklingMachine,machineParameters.getUserNameForMachine( sourceMachines[i] ) , sourceMachines[i] , remotePxLogPath, paths.STATSLOGS, sourceMachines[i] ) print output else: for j in range(3):#do 3 times in case of currently turning log files. remotePxLogPath = paths.getPXPathFromMachine(paths.PXLOG, sourceMachines[i], machineParameters.getUserNameForMachine( sourceMachines[i])) print "le remote paths : ", remotePxLogPath print output = commands.getoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( machineParameters.getUserNameForMachine( sourceMachines[i] ), sourceMachines[i] , remotePxLogPath, paths.STATSLOGS, sourceMachines[i] ) ) print print "*****************************" print print "rsync -avzr --delete-before -e ssh " + """%s@%s:%s %s%s/""" %( machineParameters.getUserNameForMachine( sourceMachines[i] ), sourceMachines[i] , remotePxLogPath, paths.STATSLOGS, sourceMachines[i] ) print print output if picklingMachine != LOCAL_MACHINE :#pickling to be done elsewhere,needs ssh output = commands.getoutput( """ssh %s@%s 'python %spickleUpdater.py -m %s -f rx --date "%s" ' """ %( machineParameters.getUserNameForMachine( picklingMachine ), picklingMachine, paths.STATSBIN, sourceMachines[i], currentTimeInIsoFormat ) ) print "ssh %s@%s 'python %spickleUpdater.py -m %s -f rx' " %( machineParameters.getUserNameForMachine( picklingMachine ), picklingMachine, paths.STATSBIN, sourceMachines[i] ) print output output = commands.getoutput( """ssh %s@%s 'python %spickleUpdater.py -m %s -f tx --date "%s" ' """( machineParameters.getUserNameForMachine( picklingMachine ), picklingMachine , paths.STATSBIN, sourceMachines[i], currentTimeInIsoFormat ) ) print "ssh %s@%s 'python %spickleUpdater.py -m %s -f tx' "%( machineParameters.getUserNameForMachine( picklingMachine ), picklingMachine , paths.STATSBIN, sourceMachines[i] ) print output output = commands.getoutput( """%spickleSynchroniser.py -l %s -m %s """%( paths.STATSTOOLS, machineParameters.getUserNameForMachine( picklingMachine ), picklingMachine ) ) print "%spickleSynchroniser.py -l %s -m %s " %( paths.STATSTOOLS, machineParameters.getUserNameForMachine( picklingMachine ), picklingMachine ) print output else: # pickling is to be done locally. Log files may or may not reside elsewhere. output = commands.getoutput( """python %spickleUpdater.py -f rx -m %s --date "%s" """%( paths.STATSBIN, sourceMachines[i], currentTimeInIsoFormat ) ) print "python %spickleUpdater.py -f rx -m %s " %( paths.STATSBIN, sourceMachines[i] ) print output output = commands.getoutput( """python %spickleUpdater.py -f tx -m %s --date "%s" """ %( paths.STATSBIN, sourceMachines[i], currentTimeInIsoFormat ) ) print "python %spickleUpdater.py -f tx -m %s " %( paths.STATSBIN, sourceMachines[i] ) print output sys.exit() elif nbChildProcess!=0 and nbChildProcess%3 == 0 : while True:#wait on all non terminated child process' try: #will raise exception when no child process remain. pid, status = os.wait() except: break while True:#wait on all non terminated child process' try: #will raise exception when no child process remain. pid, status = os.wait( ) except: break
def updatePickles(parameters, machineParameters, currentTimeInIsoFormat): """ @summary : Updates the pickle files for all the specified log machines so that they are available for graphic production. @note : Pickling is to be done on specified pickling machines. All the pickle files that are produced on remote machines will be downloaded on the local machine. @param parameters: StatsConfigParameters instance containing the parameters found in the config file. @param machineParameters: MachineConfigParameters instance containing the parameters found in the config file. @param currentTimeInIsoFormat : Time at which this program was originally called. """ nbChildProcess = 0 paths = StatsPaths() paths.setPaths() for tag in parameters.sourceMachinesTags: pid = os.fork() if pid == 0: #if child sourceMachines = machineParameters.getMachinesAssociatedWith(tag) for i in range(len(sourceMachines)): picklingMachine = parameters.detailedParameters.picklingMachines[ tag][i] # If pickling and source machines differ, download log files frm source to pickling machine. if sourceMachines[i] != picklingMachine: if parameters.detailedParameters.picklingMachines[tag][ i] != LOCAL_MACHINE: #pickling to be done elsewhere for j in range( 3 ): #do 3 times in case of currently turning log files. remotePxLogPath = paths.getPXPathFromMachine( paths.PXLOG, sourceMachines[i], machineParameters.getUserNameForMachine( sourceMachines[i])) output = commands.getoutput( "ssh %s@%s 'rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/' " % (machineParameters.getUserNameForMachine( picklingMachine), picklingMachine, machineParameters.getUserNameForMachine( sourceMachines[i]), sourceMachines[i], remotePxLogPath, paths.STATSLOGS, sourceMachines[i])) print "ssh %s@%s 'rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/' " % ( machineParameters.getUserNameForMachine( picklingMachine), picklingMachine, machineParameters.getUserNameForMachine( sourceMachines[i]), sourceMachines[i], remotePxLogPath, paths.STATSLOGS, sourceMachines[i]) print output else: for j in range( 3 ): #do 3 times in case of currently turning log files. remotePxLogPath = paths.getPXPathFromMachine( paths.PXLOG, sourceMachines[i], machineParameters.getUserNameForMachine( sourceMachines[i])) print "le remote paths : ", remotePxLogPath print output = commands.getoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " % (machineParameters.getUserNameForMachine( sourceMachines[i]), sourceMachines[i], remotePxLogPath, paths.STATSLOGS, sourceMachines[i])) print print "*****************************" print print "rsync -avzr --delete-before -e ssh " + """%s@%s:%s %s%s/""" % ( machineParameters.getUserNameForMachine( sourceMachines[i]), sourceMachines[i], remotePxLogPath, paths.STATSLOGS, sourceMachines[i]) print print output if picklingMachine != LOCAL_MACHINE: #pickling to be done elsewhere,needs ssh output = commands.getoutput( """ssh %s@%s 'python %spickleUpdater.py -m %s -f rx --date "%s" ' """ % (machineParameters.getUserNameForMachine( picklingMachine), picklingMachine, paths.STATSBIN, sourceMachines[i], currentTimeInIsoFormat)) print "ssh %s@%s 'python %spickleUpdater.py -m %s -f rx' " % ( machineParameters.getUserNameForMachine( picklingMachine), picklingMachine, paths.STATSBIN, sourceMachines[i]) print output output = commands.getoutput( """ssh %s@%s 'python %spickleUpdater.py -m %s -f tx --date "%s" ' """ ( machineParameters.getUserNameForMachine( picklingMachine), picklingMachine, paths.STATSBIN, sourceMachines[i], currentTimeInIsoFormat)) print "ssh %s@%s 'python %spickleUpdater.py -m %s -f tx' " % ( machineParameters.getUserNameForMachine( picklingMachine), picklingMachine, paths.STATSBIN, sourceMachines[i]) print output output = commands.getoutput( """%spickleSynchroniser.py -l %s -m %s """ % (paths.STATSTOOLS, machineParameters.getUserNameForMachine( picklingMachine), picklingMachine)) print "%spickleSynchroniser.py -l %s -m %s " % ( paths.STATSTOOLS, machineParameters.getUserNameForMachine( picklingMachine), picklingMachine) print output else: # pickling is to be done locally. Log files may or may not reside elsewhere. output = commands.getoutput( """python %spickleUpdater.py -f rx -m %s --date "%s" """ % (paths.STATSBIN, sourceMachines[i], currentTimeInIsoFormat)) print "python %spickleUpdater.py -f rx -m %s " % ( paths.STATSBIN, sourceMachines[i]) print output output = commands.getoutput( """python %spickleUpdater.py -f tx -m %s --date "%s" """ % (paths.STATSBIN, sourceMachines[i], currentTimeInIsoFormat)) print "python %spickleUpdater.py -f tx -m %s " % ( paths.STATSBIN, sourceMachines[i]) print output sys.exit() elif nbChildProcess != 0 and nbChildProcess % 3 == 0: while True: #wait on all non terminated child process' try: #will raise exception when no child process remain. pid, status = os.wait() except: break while True: #wait on all non terminated child process' try: #will raise exception when no child process remain. pid, status = os.wait() except: break