def copyFromAFS(self):
print 'afs file ...'
dest = os.path.join(self.destDir,
os.path.basename(self.name))
cmd = 'cp '+filename +' '+dest
if not os.path.exists(self.name):
self.dataset.copyStatus = 'error'
self.dataset.copyInfo = 'file does not exist :'+self.ename
elif not os.path.exists(dest):
self.dataset.copyStatus = 'success'
self.dataset.copyInfo = 'File copied to \n'
self.dataset.copyInfo += str(dest)
print 'executing :'+cmd
print 'please wait...'
runCmd = Popen4(cmd)
answer = str(runCmd.fromchild.read())
print answer
pattern ='(E|e)+rror|not found'
if re.search(pattern, answer):
Popen4('rm -f '+dest)
self.dataset.copyStatus ='error'
self.dataset.copyInfo = answer
else:
self.dataset.copyStatus = 'success'
self.dataset.copyInfo = 'File already exists '
def copyFromCastor(self):
print 'castor file...'
dest = os.path.join(self.destDir,
os.path.basename(self.name))
cmd = 'globus-url-copy gsiftp://castorgrid.cern.ch'
cmd += self.name +' '+'file://'+dest
if not os.path.exists(dest):
self.dataset.copyStatus = 'success'
self.dataset.copyInfo = 'File copied to \n'
self.dataset.copyInfo += str(dest)
print 'executing :'+cmd
print 'please wait...'
runCmd =Popen4(cmd)
answer = str(runCmd.fromchild.read())
print answer
pattern ='(E|e)+rror|not found'
if re.search(pattern, answer):
Popen4('rm -f '+dest)
self.dataset.copyStatus ='error'
self.dataset.copyInfo = answer
else:
self.dataset.copyStatus = 'success'
self.dataset.copyInfo = 'File already exists '
def setAreas(self):
if self.dataset.destDir.startswith('/castor'):
self.stageArea = os.path.join(self.dataset.stageArea,
self.name)
print 'stage Area :'+self.stageArea
self.castorArea = os.path.join(self.dataset.destDir,
self.name)
print 'castor area '+self.castorArea
Popen4('mkdir '+self.stageArea)
Popen4('rfmkdir '+self.castorArea)
return (self.stageArea, self.castorArea)
def doIt(self):
filename = string.strip(self.name)
destDir = self.destDir
if not self.isCastorFile():
print 'Not a castor file!!'
self.dataset.copyStatus = 'error'
self.dataset.copyInfo = 'Not a castor file!!'
else:
dest = os.path.join(destDir, os.path.basename(filename))
cmd = '/usr/local/lcg/edg/bin/'
cmd += 'edg-gridftp-ls gsiftp://castorgrid.cern.ch'
cmd += filename
if not os.path.exists(dest):
self.dataset.copyStatus = 'success'
self.dataset.copyInfo = 'File copied to \n'
self.dataset.copyInfo += str(dest)
print 'executing :' + cmd
print 'please wait...'
runCmd = Popen4(cmd)
answer = str(runCmd.fromchild.read())
answer = string.strip(answer)
print answer
if answer != filename:
self.dataset.copyStatus = 'error'
self.dataset.copyInfo = answer
else:
self.dataset.copyStatus = 'success'
self.dataset.copyInfo = 'File already exists '
def _ipcheck( self ):
cmd = 'ipcheck -p ' + properties.ETC_DIR
# Ignore errors the first time after startup.
if self.update_count == 0:
cmd += ' -e'
if self._is_debug():
cmd += ' -dv'
cmd += ' %s %s %s' % (self.ddns_acct, self.ddns_pswd, self.host_name)
# Start the ip checker
self.msglog( "running %s" % cmd )
child = Popen4( cmd )
self.ipcheck_pid = child.pid
outfile = child.fromchild
# Wait for ip checker to finish. Log any output in the message log.
while 1:
result = select( [outfile], [], [], 3.0 )
if result[0]:
lines = outfile.readlines()
for line in lines:
self.msglog( line )
status = child.poll()
if not status == -1:
break
self.ipcheck_pid = 0
if os.WIFEXITED( status ):
exit_code = os.WEXITSTATUS( status )
self.msglog( 'ipcheck exit status = %d' % exit_code )
else:
self.msglog( 'ipcheck forcibly stopped, status = %d' % status )
self.update_count += 1
def getPopen4Output(cmd):
po = Popen4(cmd)
out = []
while po.poll() == -1:
out.extend(po.fromchild.readlines())
del po
return out
def run(cmd, options):
"""Run command in a sub-process, capturing the output (stdout and stderr).
Return the exist code, and output."""
# TODO: Use subprocess.Popen if we're running on Python 2.4
if os.name == 'nt' or sys.platform.startswith('java'):
tochild, fromchild = os.popen4(cmd)
tochild.close()
output = fromchild.read()
ret = fromchild.close()
if ret == None:
ret = 0
else:
proc = Popen4(cmd)
try:
output = ''
proc.tochild.close()
output = proc.fromchild.read()
ret = proc.wait()
if os.WIFEXITED(ret):
ret = os.WEXITSTATUS(ret)
except Timeout:
vlog('# Process %d timed out - killing it' % proc.pid)
os.kill(proc.pid, signal.SIGTERM)
ret = proc.wait()
if ret == 0:
ret = signal.SIGTERM << 8
output += ("\n### Abort: timeout after %d seconds.\n"
% options.timeout)
return ret, splitnewlines(output)
def run(self):
"""start the command"""
# print "Starting ",self.cmdline
self.resStart=getrusage(self.who)
self.timeStart=time()
if sys.version_info<(2,4):
run=Popen4(self.cmdline)
self.output=run.fromchild
else:
run=subprocess.Popen(shellExecutionPrefix()+self.cmdline,
shell=True,
bufsize=0,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True)
self.output=run.stdout
self.run=run
self.threadPid=run.pid
foamLogger().info("Started with PID %d" % self.threadPid)
if self.isLinux:
# print "Starting Timer"
self.timer=Timer(0.1*self.timerTime,getLinuxMem,args=[self])
self.timer.start()
# print "Starting Timer"
self.timer2=Timer(0.5*self.timerTime,checkForStopFile,args=[self])
self.timer2.start()
self.hasSomethingToSay=True
self.stateLock.release()
try:
# print "Waiting",time()
self.status=run.wait()
# Python 2.3 on Mac OS X never seems to reach this point
# print "After wait",time()
# print "Status:",self.status
# to give a chance to read the remaining output
if self.hasSomethingToSay:
sleep(2.)
while self.reader.read(self.output):
print_("Unused output:",self.reader.line)
except OSError:
e = sys.exc_info()[1] # compatible with 2.x and 3.x
print_("Exeption caught:",e)
self.stopTimer()
self.threadPid=-1
self.resEnd=getrusage(self.who)
self.timeEnd=time()
# print "End:",self.timeEnd
# print "Returned",self.status
self.getReturnCode()
def assertPosixSubprocess(self, cmd):
cmd = self.locate_command(cmd)
process = Popen4(cmd, env=os.environ)
process.tochild.close()
output = process.fromchild.read()
status = process.wait()
process.fromchild.close()
return status, output
def getPopen4Output(cmd, max_lines=-1):
po = Popen4(cmd)
out = []
while po.poll() == -1:
out.extend(po.fromchild.readlines())
if max_lines > 0 and length(out) >= max_lines: break
del po
return out
def getDatasetNames(self):
#returns a list of datasets on disk
ls_Dir = Popen4('ls '+self.dir)
data = ls_Dir.fromchild.readlines()
self.datasets=[]
for ds in data:
ds = os.path.join(self.dir, ds)
self.datasets.append(ds[0:-1])
def assertPosixSubprocess(self, cmd):
cmd = cmd.replace('cheetah', self.cmd)
process = Popen4(cmd, env=os.environ)
process.tochild.close()
output = process.fromchild.read()
status = process.wait()
process.fromchild.close()
return status, output
def mkDestDir(self):
loc = os.path.join(self.destDir, self.name)
if not os.path.exists(loc):
cmd = 'mkdir ' + loc
runCmd = Popen4(cmd)
runCmd.wait()
self.destDir = loc
print 'Datasets dir: ' + self.destDir
def doCopy(self):
self.mkDestDir()
cmd = 'dq2_get -s ' + self.dataset.site + ' -r -d ' + self.destDir
cmd += ' ' + self.name
cmd += self.filesString()
print 'executing :' + cmd
print 'please wait...'
runCmd = Popen4(cmd)
answer = runCmd.fromchild.readlines()
self.handleCmdOutput(answer)
def doCopy(self):
cmd = 'dq2_get -rc -d ' + self.destDir
cmd += ' ' + self.name
cmd += self.filesString()
print 'executing :' + cmd
print 'please wait...'
runCmd = Popen4(cmd)
answer = str(runCmd.fromchild.read())
answer = string.strip(answer)
print answer
def execute(self, *args):
process = Popen4([self.facility.path] + self.facility.args +
list(args))
timers = [
Timer(time, self.kill, [process.pid, signal])
for time, signal in self.facility.wait.iteritems()
]
for timer in timers:
timer.start()
status = process.wait()
for timer in timers:
# No penalty, btw, for cancelling a dead timer
if timer.isAlive():
timer.cancel()
process.tochild.close()
if __debug__:
while True:
line = process.fromchild.readline()
if line:
syslog(line)
else:
break
process.fromchild.close()
command = basename(self.facility.path)
if WIFEXITED(status):
exit_status = WEXITSTATUS(status)
if exit_status != EX_OK:
raise ExecutionError(
EX_SOFTWARE, '`%(command)s\' exited with \
error-code %(exit_status)d' % {
'command': command,
'exit_status': exit_status
})
elif WIFSIGNALED(status):
raise ExecutionError(
EX_SOFTWARE, '`%(command)s\' terminated \
with signal %(signal)d' % {
'command': command,
'signal': WTERMSIG(status)
})
elif WIFSTOPPED(status):
raise ExecutionError(
EX_SOFTWARE, '`%(command)s\' stopped with \
signal %(signal)d' % {
'command': command,
'signal': WSTOPSIG(status)
})
else:
# Failsafe: timers should have killed the process by this point, or
# it should have ended naturally.
kill(process.pid, SIGKILL)
raise ExecutionError(
EX_SOFTWARE, 'Failed timer on `%(command)s\'; \
terminating the process extraordinarily.' % {'command': command})
def schedule(self, job):
import pyre.util as util
# Fix-up the job.
if not job.task:
job.task = "jobname"
job.walltime = util.hms(job.dwalltime.value)
job.arguments = ' '.join(job.arguments)
job.resourceList = self.buildResourceList(job)
# Generate the main PBS batch script.
script = self.retrieveTemplate('batch.sh', ['schedulers', 'scripts', self.name])
if script is None:
self._error.log("could not locate batch script template for '%s'" % self.name)
sys.exit(1)
script.scheduler = self
script.job = job
if self.dry:
print script
return
try:
import os
from popen2 import Popen4
cmd = [self.command]
self._info.log("spawning: %s" % ' '.join(cmd))
child = Popen4(cmd)
self._info.log("spawned process %d" % child.pid)
print >> child.tochild, script
child.tochild.close()
for line in child.fromchild:
self._info.line(" " + line.rstrip())
status = child.wait()
self._info.log()
exitStatus = None
if (os.WIFSIGNALED(status)):
statusStr = "signal %d" % os.WTERMSIG(status)
elif (os.WIFEXITED(status)):
exitStatus = os.WEXITSTATUS(status)
statusStr = "exit %d" % exitStatus
else:
statusStr = "status %d" % status
self._info.log("%s: %s" % (cmd[0], statusStr))
except IOError, e:
self._error.log("%s: %s" % (self.command, e))
return
def _popen(self, command, paths, fault):
"""Given a website ID, a command and a Fault template, run the command.
"""
process = Popen4(command)
result = process.wait()
output = process.fromchild.read()
if result != 0:
message = fault[1] % (paths['repo'], result, output)
return Fault(fault[0], message)
return result
def get_image_size(self, file_path):
"""Probe the size of an on-disk image, returning a (width, height) tuple.
"""
child = Popen4("%s -ping %s" % (self.IDENTIFY, commands.mkarg(file_path)))
output = child.fromchild.read()
status = child.wait()
match = re.search(r" (\d+)x(\d+) ", output)
if status or not match:
raise ImageException(output)
return int(match.group(1)), int(match.group(2))
def doCopy(self):
self.cpstatus = {}
stage,castor = self.setAreas()
print 'stage ' +stage
print 'castor ' +castor
files = [os.path.basename(file) for file in self.dataset.filesToCopy]
for file in files:
dq2Getcmd = 'dq2_get -s '+self.dataset.site
dq2Getcmd += ' -rc -d '+stage
dq2Getcmd += ' '+self.name
dq2Getcmd += ' '+file
temp = os.path.join(stage, file)
dest = os.path.join(castor, file)
try:
cmd = Popen4('rfdir '+dest)
result = str(cmd.fromchild.read())
if re.search('No such file', result):
print dq2Getcmd
fromGrid = Popen4(dq2Getcmd)
answer = fromGrid.fromchild.readlines()
self.handleStatus(self.name, answer, self.cpstatus)
#fromGrid.wait()
toCastor = Popen4('rfcp '+temp+ ' '+dest)
toCastor.wait()
else:
print 'File already available on castor'
print 'src file '+temp
print 'dest file '+dest
time.sleep(10)
rmtemp = Popen4('rm -f '+temp)
rmtemp.wait()
print 'deleted '+temp
except Exception, e:
print 'problem getting data from grid %s'%str(e)
def setProxyDetails(self):
dict = {}
lines = Popen4('grid-proxy-info ').fromchild.readlines()
lines = [line.strip() for line in lines]
for line in lines:
parts = line.split(':')
if len(parts) < 2:
pass
else:
dict[parts[0].strip()] = parts[1].strip()
self.__dict__.update(dict)
def thumbnail_image(self, src_path, dest_path, size):
"""Generate a square thumbnail, with on-disk source and destination paths."""
child = Popen4("%s -geometry %dx%d %s %s" % (
self.CONVERT,
size, size,
commands.mkarg(src_path),
commands.mkarg(dest_path),
))
output = child.fromchild.read()
status = child.wait()
if status:
raise ImageException(output)
def execute_command(command):
sub_cmd = Popen4(command)
cmd_output = sub_cmd.fromchild.readlines()
status = sub_cmd.wait()
if os.WIFEXITED(status):
exit_status = os.WEXITSTATUS(status)
if exit_status:
return 0, cmd_output
else:
raise Exception('execute_command failure. Reason: %s.\n' %
os.strerror(status))
return 1, cmd_output
def run(self, command):
"""
Execute a command in a child process and return the output.
@type command C{str}
@param command: The shell command to run.
@raise L{RuntimeError}: If the child process exits with an error.
"""
process = Popen4(command)
stdout = process.fromchild.read()
exitCode = process.wait()
if os.WIFSIGNALED(exitCode) or os.WEXITSTATUS(exitCode):
raise CommandFailed(exitCode, stdout)
return stdout
def doIt(self):
filename = string.strip(self.name)
destDir = self.destDir
#cmd = '/usr/local/lcg/edg/bin/'
cmd += 'edg-gridftp-ls gsiftp://castorgrid.cern.ch'
cmd += filename
print 'executing :'+cmd
print 'please wait...'
runCmd = Popen4(cmd)
answer = str(runCmd.fromchild.read())
answer = string.strip(answer)
print answer
if answer != filename :
self.dataset.copyStatus = 'error'
self.dataset.copyInfo = answer
def _extract(self, releaseFileName, distFileName, pkgName, uid):
pextract = find_tool(releaseFileName, 'pextract')
if not pextract:
self.completionStatus = (1, self.etitle,
"Unable to locate pextract tool.\n"
"Check config options and try again.")
return
cmd = os.path.join(pextract, 'pextract')
if uid:
cmd += " -u %d" % uid
cmd += " -d %s %s %s" % (distFileName, releaseFileName, pkgName)
child = Popen4(cmd)
outfile = child.fromchild
progress = 0
self._set_progress(progress, "Creating distribution")
self._proc_message("Executing %s\n" % cmd)
try:
while 1:
result = select([outfile], [], [])
if result[0]:
line = outfile.readline()
if line:
self._proc_message(line)
if progress < 95:
progress += 5
self._set_progress(progress, "Creating distribution")
else:
self.completionStatus = (0, None,
"Extraction complete.")
break
self._set_progress(100)
status = child.wait()
if os.WIFEXITED(status):
exit_code = os.WEXITSTATUS(status)
if exit_code:
self.completionStatus = (1, self.etitle,
os.strerror(exit_code))
else:
self.completionStatus = (
1, self.etitle,
"Extraction forcibly stopped, status = %d" % status)
except KeyboardInterrupt:
os.kill(child.pid, signal.SIGTERM)
self._proc_message("Extraction cancelled by user request.\n")
def fire(self, what):
"""
Submits the job by spawning a subprocess.
The Popen objects captures std error, and std out
'what' is a a key in to tthe self.scripts dictionary
the popen object is bound to the object ot allow subsequent
interogation. This is a specificity of interactive running.
"""
cmd = self.scripts[what]['cmd']
m = 'fire commandL\n: %s' % cmd
self.logger.debug(m)
self.popen = Popen4(cmd)
self.subLog = self.popen.fromchild.readlines()
def timed_cmd(c, secs=-1, execute=1):
def isTimedOut(start, secs):
if secs == -1: return False
return (time() - start) > secs
from time import time
from popen2 import Popen4
po = Popen4(c)
proc = po.pid
out = []
start = time()
while po.poll() == -1:
if isTimedOut(start, secs):
m = 'Command timed out(%d secs)!\n%s' % (secs, c)
raise RTTTimeOut(m)
out.extend(po.fromchild.readlines())
del po
return out
def brief_diff(source_tag):
"""
Unit Tests
----------
>>> x = CVS_Mixin()
>>> r = x.brief_diff()
Traceback (most recent call last):
...
TypeError: brief_diff() takes exactly 1 argument (0 given)
>>> r = x.brief_diff(None, None)
Traceback (most recent call last):
...
TypeError: brief_diff() takes exactly 1 argument (2 given)
"""
command = ('cvs diff -N --brief -r %s' % source_tag)
child = Popen4(command)
result = child.fromchild.readlines()
child.wait()
return result
def runCommand(args):
"""
Execute a vector of arguments.
@type args: C{list} of C{str}
@param args: A list of arguments, the first of which will be used as the
executable to run.
@rtype: C{str}
@return: All of the standard output.
@raise CommandFailed: when the program exited with a non-0 exit code.
"""
process = Popen4(args)
stdout = process.fromchild.read()
exitCode = process.wait()
if os.WIFSIGNALED(exitCode) or os.WEXITSTATUS(exitCode):
raise CommandFailed(exitCode, stdout)
return stdout
def _run_child(self, cmd):
"Set process group and run child."
os.setpgrp()
Popen4._run_child(self, cmd)
def __init__(self, cmd, bufsize=-1):
Popen4.__init__(self, cmd, bufsize)
def __init__(self, cmd, bufsize=-1, env=os.environ):
self._env = env
Popen4.__init__(self, cmd, bufsize)