def transferLogFiles():
"""
@summary : Log files will not be tansferred if local machine
is not designed to be a pickling machine.
If log files are to be transferred, they will be straight
from the source."
"""
paths = StatsPaths()
paths.setPaths()
parameters = StatsConfigParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
parameters.getAllParameters()
individualSourceMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.sourceMachinesTags )
individualPicklingMachines = machineParameters.getMachinesAssociatedWithListOfTags( parameters.picklingMachines )
for sourceMachine,picklingMachine in map( None, individualSourceMachines, individualPicklingMachines ) :
if picklingMachine == LOCAL_MACHINE :#pickling to be done here
userName = machineParameters.getUserNameForMachine(sourceMachine)
remoteLogPath = paths.getPXPathFromMachine( paths.PXLOG, sourceMachine, userName )
print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( userName , sourceMachine,remoteLogPath , paths.STATSLOGS, sourceMachine )
output = commands.getoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( userName , sourceMachine, remoteLogPath, paths.STATSLOGS, sourceMachine ) )
print output
def transferLogFiles():
"""
@summary : Log files will not be tansferred if local machine
is not designed to be a pickling machine.
If log files are to be transferred, they will be straight
from the source."
"""
paths = StatsPaths()
paths.setPaths()
parameters = StatsConfigParameters()
machineParameters = MachineConfigParameters()
machineParameters.getParametersFromMachineConfigurationFile()
parameters.getAllParameters()
individualSourceMachines = machineParameters.getMachinesAssociatedWithListOfTags(
parameters.sourceMachinesTags)
individualPicklingMachines = machineParameters.getMachinesAssociatedWithListOfTags(
parameters.picklingMachines)
for sourceMachine, picklingMachine in map(None, individualSourceMachines,
individualPicklingMachines):
if picklingMachine == LOCAL_MACHINE: #pickling to be done here
userName = machineParameters.getUserNameForMachine(sourceMachine)
remoteLogPath = paths.getPXPathFromMachine(paths.PXLOG,
sourceMachine, userName)
print "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " % (
userName, sourceMachine, remoteLogPath, paths.STATSLOGS,
sourceMachine)
output = commands.getoutput(
"rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %
(userName, sourceMachine, remoteLogPath, paths.STATSLOGS,
sourceMachine))
print output
def updatePickles( parameters, machineParameters, currentTimeInIsoFormat ):
"""
@summary : Updates the pickle files for all the specified log machines
so that they are available for graphic production.
@note : Pickling is to be done on specified pickling machines.
All the pickle files that are produced on remote machines will be
downloaded on the local machine.
@param parameters: StatsConfigParameters instance containing
the parameters found in the config file.
@param machineParameters: MachineConfigParameters instance containing
the parameters found in the config file.
@param currentTimeInIsoFormat : Time at which this program was originally
called.
"""
nbChildProcess = 0
paths = StatsPaths()
paths.setPaths()
for tag in parameters.sourceMachinesTags:
pid = os.fork()
if pid == 0: #if child
sourceMachines = machineParameters.getMachinesAssociatedWith(tag)
for i in range( len( sourceMachines ) ):
picklingMachine = parameters.detailedParameters.picklingMachines[tag][i]
# If pickling and source machines differ, download log files frm source to pickling machine.
if sourceMachines[i] != picklingMachine:
if parameters.detailedParameters.picklingMachines[tag][i] != LOCAL_MACHINE :#pickling to be done elsewhere
for j in range(3):#do 3 times in case of currently turning log files.
remotePxLogPath = paths.getPXPathFromMachine(paths.PXLOG, sourceMachines[i], machineParameters.getUserNameForMachine( sourceMachines[i]))
output = commands.getoutput( "ssh %s@%s 'rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/' " %( machineParameters.getUserNameForMachine( picklingMachine), picklingMachine,machineParameters.getUserNameForMachine( sourceMachines[i] ) , sourceMachines[i] , remotePxLogPath, paths.STATSLOGS, sourceMachines[i] ) )
print "ssh %s@%s 'rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/' " %( machineParameters.getUserNameForMachine( picklingMachine), picklingMachine,machineParameters.getUserNameForMachine( sourceMachines[i] ) , sourceMachines[i] , remotePxLogPath, paths.STATSLOGS, sourceMachines[i] )
print output
else:
for j in range(3):#do 3 times in case of currently turning log files.
remotePxLogPath = paths.getPXPathFromMachine(paths.PXLOG, sourceMachines[i], machineParameters.getUserNameForMachine( sourceMachines[i]))
print "le remote paths : ", remotePxLogPath
print
output = commands.getoutput( "rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ " %( machineParameters.getUserNameForMachine( sourceMachines[i] ), sourceMachines[i] , remotePxLogPath, paths.STATSLOGS, sourceMachines[i] ) )
print
print "*****************************"
print
print "rsync -avzr --delete-before -e ssh " + """%s@%s:%s %s%s/""" %( machineParameters.getUserNameForMachine( sourceMachines[i] ), sourceMachines[i] , remotePxLogPath, paths.STATSLOGS, sourceMachines[i] )
print
print output
if picklingMachine != LOCAL_MACHINE :#pickling to be done elsewhere,needs ssh
output = commands.getoutput( """ssh %s@%s 'python %spickleUpdater.py -m %s -f rx --date "%s" ' """ %( machineParameters.getUserNameForMachine( picklingMachine ), picklingMachine, paths.STATSBIN, sourceMachines[i], currentTimeInIsoFormat ) )
print "ssh %s@%s 'python %spickleUpdater.py -m %s -f rx' " %( machineParameters.getUserNameForMachine( picklingMachine ), picklingMachine, paths.STATSBIN, sourceMachines[i] )
print output
output = commands.getoutput( """ssh %s@%s 'python %spickleUpdater.py -m %s -f tx --date "%s" ' """( machineParameters.getUserNameForMachine( picklingMachine ), picklingMachine , paths.STATSBIN, sourceMachines[i], currentTimeInIsoFormat ) )
print "ssh %s@%s 'python %spickleUpdater.py -m %s -f tx' "%( machineParameters.getUserNameForMachine( picklingMachine ), picklingMachine , paths.STATSBIN, sourceMachines[i] )
print output
output = commands.getoutput( """%spickleSynchroniser.py -l %s -m %s """%( paths.STATSTOOLS, machineParameters.getUserNameForMachine( picklingMachine ), picklingMachine ) )
print "%spickleSynchroniser.py -l %s -m %s " %( paths.STATSTOOLS, machineParameters.getUserNameForMachine( picklingMachine ), picklingMachine )
print output
else: # pickling is to be done locally. Log files may or may not reside elsewhere.
output = commands.getoutput( """python %spickleUpdater.py -f rx -m %s --date "%s" """%( paths.STATSBIN, sourceMachines[i], currentTimeInIsoFormat ) )
print "python %spickleUpdater.py -f rx -m %s " %( paths.STATSBIN, sourceMachines[i] )
print output
output = commands.getoutput( """python %spickleUpdater.py -f tx -m %s --date "%s" """ %( paths.STATSBIN, sourceMachines[i], currentTimeInIsoFormat ) )
print "python %spickleUpdater.py -f tx -m %s " %( paths.STATSBIN, sourceMachines[i] )
print output
sys.exit()
elif nbChildProcess!=0 and nbChildProcess%3 == 0 :
while True:#wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait()
except:
break
while True:#wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait( )
except:
break
def updatePickles(parameters, machineParameters, currentTimeInIsoFormat):
"""
@summary : Updates the pickle files for all the specified log machines
so that they are available for graphic production.
@note : Pickling is to be done on specified pickling machines.
All the pickle files that are produced on remote machines will be
downloaded on the local machine.
@param parameters: StatsConfigParameters instance containing
the parameters found in the config file.
@param machineParameters: MachineConfigParameters instance containing
the parameters found in the config file.
@param currentTimeInIsoFormat : Time at which this program was originally
called.
"""
nbChildProcess = 0
paths = StatsPaths()
paths.setPaths()
for tag in parameters.sourceMachinesTags:
pid = os.fork()
if pid == 0: #if child
sourceMachines = machineParameters.getMachinesAssociatedWith(tag)
for i in range(len(sourceMachines)):
picklingMachine = parameters.detailedParameters.picklingMachines[
tag][i]
# If pickling and source machines differ, download log files frm source to pickling machine.
if sourceMachines[i] != picklingMachine:
if parameters.detailedParameters.picklingMachines[tag][
i] != LOCAL_MACHINE: #pickling to be done elsewhere
for j in range(
3
): #do 3 times in case of currently turning log files.
remotePxLogPath = paths.getPXPathFromMachine(
paths.PXLOG, sourceMachines[i],
machineParameters.getUserNameForMachine(
sourceMachines[i]))
output = commands.getoutput(
"ssh %s@%s 'rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/' "
% (machineParameters.getUserNameForMachine(
picklingMachine), picklingMachine,
machineParameters.getUserNameForMachine(
sourceMachines[i]), sourceMachines[i],
remotePxLogPath, paths.STATSLOGS,
sourceMachines[i]))
print "ssh %s@%s 'rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/' " % (
machineParameters.getUserNameForMachine(
picklingMachine), picklingMachine,
machineParameters.getUserNameForMachine(
sourceMachines[i]), sourceMachines[i],
remotePxLogPath, paths.STATSLOGS,
sourceMachines[i])
print output
else:
for j in range(
3
): #do 3 times in case of currently turning log files.
remotePxLogPath = paths.getPXPathFromMachine(
paths.PXLOG, sourceMachines[i],
machineParameters.getUserNameForMachine(
sourceMachines[i]))
print "le remote paths : ", remotePxLogPath
print
output = commands.getoutput(
"rsync -avzr --delete-before -e ssh %s@%s:%s %s%s/ "
% (machineParameters.getUserNameForMachine(
sourceMachines[i]), sourceMachines[i],
remotePxLogPath, paths.STATSLOGS,
sourceMachines[i]))
print
print "*****************************"
print
print "rsync -avzr --delete-before -e ssh " + """%s@%s:%s %s%s/""" % (
machineParameters.getUserNameForMachine(
sourceMachines[i]), sourceMachines[i],
remotePxLogPath, paths.STATSLOGS,
sourceMachines[i])
print
print output
if picklingMachine != LOCAL_MACHINE: #pickling to be done elsewhere,needs ssh
output = commands.getoutput(
"""ssh %s@%s 'python %spickleUpdater.py -m %s -f rx --date "%s" ' """
% (machineParameters.getUserNameForMachine(
picklingMachine), picklingMachine, paths.STATSBIN,
sourceMachines[i], currentTimeInIsoFormat))
print "ssh %s@%s 'python %spickleUpdater.py -m %s -f rx' " % (
machineParameters.getUserNameForMachine(
picklingMachine), picklingMachine, paths.STATSBIN,
sourceMachines[i])
print output
output = commands.getoutput(
"""ssh %s@%s 'python %spickleUpdater.py -m %s -f tx --date "%s" ' """ (
machineParameters.getUserNameForMachine(
picklingMachine), picklingMachine,
paths.STATSBIN, sourceMachines[i],
currentTimeInIsoFormat))
print "ssh %s@%s 'python %spickleUpdater.py -m %s -f tx' " % (
machineParameters.getUserNameForMachine(
picklingMachine), picklingMachine, paths.STATSBIN,
sourceMachines[i])
print output
output = commands.getoutput(
"""%spickleSynchroniser.py -l %s -m %s """ %
(paths.STATSTOOLS,
machineParameters.getUserNameForMachine(
picklingMachine), picklingMachine))
print "%spickleSynchroniser.py -l %s -m %s " % (
paths.STATSTOOLS,
machineParameters.getUserNameForMachine(
picklingMachine), picklingMachine)
print output
else: # pickling is to be done locally. Log files may or may not reside elsewhere.
output = commands.getoutput(
"""python %spickleUpdater.py -f rx -m %s --date "%s" """
% (paths.STATSBIN, sourceMachines[i],
currentTimeInIsoFormat))
print "python %spickleUpdater.py -f rx -m %s " % (
paths.STATSBIN, sourceMachines[i])
print output
output = commands.getoutput(
"""python %spickleUpdater.py -f tx -m %s --date "%s" """
% (paths.STATSBIN, sourceMachines[i],
currentTimeInIsoFormat))
print "python %spickleUpdater.py -f tx -m %s " % (
paths.STATSBIN, sourceMachines[i])
print output
sys.exit()
elif nbChildProcess != 0 and nbChildProcess % 3 == 0:
while True: #wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait()
except:
break
while True: #wait on all non terminated child process'
try: #will raise exception when no child process remain.
pid, status = os.wait()
except:
break
