def run2(self,band):
"""Run SExtractor on the sciImageList in dual image mode (i.e. detection image vs filter image)"""
dname=string.split(self.detectionImage,'detection_')[0] + 'detection_' + band + '.fits'
self.detectionImage=dname
if os.path.isfile(self.detectionImage):
pass
else:
errtxt="This method is not appropriate. No Detection Image found for this observation."
self.errorList.append((self.modName,errtxt))
raise IOError,errtxt
# pdb.set_trace()
print "Run filter catalogs ..."
curdir = os.getcwd()
os.chdir(self.obsFits)
self.catalogList = []
n=-1 # num of sources, WZ
for fitsfile in self.sciImageList:
print "Generating source catalog for",fitsfile
self.logfile.write('running SExtractor on ' + fitsfile)
fitsname = string.split(fitsfile,'.')[0]
bandname = string.replace(fitsname,'sci',band) #WZ
inParFileName = os.path.join(self.obsPars, bandname +'_'+band+ '.inpar')
catalogFileName = os.path.join(self.obsCats, bandname +'_'+band+ '.cat') # by xingxing "band" added
cmd = 'sex ' + self.detectionImage + ' ' + os.path.join(self.obsFits, fitsfile) \
+ ' -c ' + inParFileName
self.skip=0 # by xingxing
try:
#print "skip in filterCatalog.py for saving time! xingxing ======IMPORTANT=======" #XX#
# if you want to skip this step, please check the commands with "#XX#"
if os.path.isfile(catalogFileName):
self.logfile.write( catalogFileName+" already exits !!!")
self.logfile.write("skip run2() in filterCatalog.py for saving time! ======IMPORTANT=======")
self.skip=1
else:
subproc = popen2.Popen4(cmd) #XX#
stderr_lines = subproc.fromchild.readlines() #XX#
# call the errorSearch ufunc and trim out the junk.
# errorSearch returns a dictionary of unique error strings and the
# number of occurrences found in the passed list (errs).
foundErrs = errorSearch(stderr_lines) # ufunc errorSearch from pUtil #XX#
if foundErrs: #XX#
self.logfile.write('SExtractor produced the following message(s) on ' + fitsfile) #XX#
for f in foundErrs.keys(): #XX#
self.logfile.write(f+": occurred "+str(foundErrs[f])+" times") #XX#
self.errorList.append((self.modName,f+": occurred "+str(foundErrs[f])+" times")) #XX#
except Exception, err:
self.logfile.write('run2 method encountered a problem: ')
self.logfile.write(str(err))
self.errorList.append((self.modName,str(err)))
raise Exception, err
if not os.path.isfile(catalogFileName):
raise RuntimeError,catalogFileName+" was not written!"
if self.skip==0: #by xingxing
self.logfile.write("Calling _magFix method")
self.logfile.write("Fixing bad mags in "+os.path.split(catalogFileName)[1])
self._magFix(catalogFileName)
#m=len(tableio.get_data(catalogFileName,1)) #WZ
#if (n < 0):
# n=m
#else:
# if (m != n):
# print "Warning: catalog dimension is different",m," ",catalogFileName
# self.logfile.write("Warning: catalog dimension is different",m," ",catalogFileName)
self.catalogList.append(catalogFileName)
self.outputList[os.path.basename(catalogFileName)] = [fitsfile]
#!/usr/bin/python
import popen2
import re
if __name__ == '__main__':
print "Try to get all assigned IP addresses:"
try:
t = re.compile("inet (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})/")
counter = 0
ipListe = []
command = "ip addr show eth0"
child = popen2.Popen4(command)
line = child.fromchild.readline()
while line:
line = str(line).strip()
match = t.search(line)
if match:
#print "\t%s" % (match.groups()[0])
ipListe.append(str(match.groups()[0]).strip())
counter += 1
line = child.fromchild.readline()
child.wait()
print counter
print ipListe[0]
except KeyboardInterrupt:
pass
def __init__(self, command, working_dir=None, capture_stderr=True, env=None):
"""Changes into a specified directory, if provided, and executes a command.
Restores the old directory afterwards.
Args:
command: The command to run, in the form of sys.argv.
working_dir: The directory to change into.
capture_stderr: Determines whether to capture stderr in the output member
or to discard it.
env: Dictionary with environment to pass to the subprocess.
Returns:
An object that represents outcome of the executed process. It has the
following attributes:
terminated_by_signal True iff the child process has been terminated
by a signal.
signal Sygnal that terminated the child process.
exited True iff the child process exited normally.
exit_code The code with which the child process exited.
output Child process's stdout and stderr output
combined in a string.
"""
# The subprocess module is the preferrable way of running programs
# since it is available and behaves consistently on all platforms,
# including Windows. But it is only available starting in python 2.4.
# In earlier python versions, we revert to the popen2 module, which is
# available in python 2.0 and later but doesn't provide required
# functionality (Popen4) under Windows. This allows us to support Mac
# OS X 10.4 Tiger, which has python 2.3 installed.
if _SUBPROCESS_MODULE_AVAILABLE:
if capture_stderr:
stderr = subprocess.STDOUT
else:
stderr = subprocess.PIPE
p = subprocess.Popen(command,
stdout=subprocess.PIPE, stderr=stderr,
cwd=working_dir, universal_newlines=True, env=env)
# communicate returns a tuple with the file obect for the child's
# output.
self.output = p.communicate()[0]
self._return_code = p.returncode
else:
old_dir = os.getcwd()
def _ReplaceEnvDict(dest, src):
# Changes made by os.environ.clear are not inheritable by child
# processes until Python 2.6. To produce inheritable changes we have
# to delete environment items with the del statement.
for key in dest.keys():
del dest[key]
dest.update(src)
# When 'env' is not None, backup the environment variables and replace
# them with the passed 'env'. When 'env' is None, we simply use the
# current 'os.environ' for compatibility with the subprocess.Popen
# semantics used above.
if env is not None:
old_environ = os.environ.copy()
_ReplaceEnvDict(os.environ, env)
try:
if working_dir is not None:
os.chdir(working_dir)
if capture_stderr:
p = popen2.Popen4(command)
else:
p = popen2.Popen3(command)
p.tochild.close()
self.output = p.fromchild.read()
ret_code = p.wait()
finally:
os.chdir(old_dir)
# Restore the old environment variables
# if they were replaced.
if env is not None:
_ReplaceEnvDict(os.environ, old_environ)
# Converts ret_code to match the semantics of
# subprocess.Popen.returncode.
if os.WIFSIGNALED(ret_code):
self._return_code = -os.WTERMSIG(ret_code)
else: # os.WIFEXITED(ret_code) should return True here.
self._return_code = os.WEXITSTATUS(ret_code)
if self._return_code < 0:
self.terminated_by_signal = True
self.exited = False
self.signal = -self._return_code
else:
self.terminated_by_signal = False
self.exited = True
self.exit_code = self._return_code
def get_snmp_value(cmd, timeout=120):
"""Run a command and spit out the output onto the web page
using the context and out descriptors.
"""
import fcntl
import popen2
import signal
import time
import select
child = None
rtnVal = ""
try:
try:
child = popen2.Popen4(cmd)
flags = fcntl.fcntl(child.fromchild, fcntl.F_GETFL)
fcntl.fcntl(child.fromchild, fcntl.F_SETFL, flags | os.O_NDELAY)
endtime = time.time() + timeout
pollPeriod = 1
firstPass = True
while time.time() < endtime and (firstPass or child.poll() == -1):
firstPass = False
r, w, e = select.select([child.fromchild], [], [], pollPeriod)
if r:
t = child.fromchild.read()
#We are sometimes getting to this point without any data
# from child.fromchild. I don't think that should happen
# but the conditional below seems to be necessary.
if t:
t = t.strip('\n')
#print t
cmds = t.split(' ')
#print cmds
snmpVal = cmds[len(cmds) - 1]
if snmpVal.count('"'):
rtnVal = snmpVal.strip('"')
else:
rtnVal = snmpVal
if child.poll() == -1:
#context.write(out,
#'Command timed out for %s' % cmd +
#' (timeout is %s seconds)' %
#timeout )
print("'Command timed out for %s" % cmd +
" (timeout is %s seconds)" % timeout)
except:
#context.write( out, 'Error running command' )
#context.write( out, 'type: %s value: %s' % tuple(sys.exc_info()[:2]))
#context.write(out, '')
print("Error running command type: %s value: %s " %
tuple(sys.exc_info()[:2]))
finally:
if child and child.poll() == -1:
os.kill(child.pid, signal.SIGKILL)
return rtnVal
def preload_linux(self):
#
#
self.reboot = 0
self.log.logp('Perform Preload in Linux')
#
os.system("rm -f PRELOAD.DON")
os.system("rm -f error.log")
os.system("rm -f ERROR.LOG")
os.system("rm -f mops*.*")
os.system("rm -f *.mop")
#
self.preload_hdd_determine()
#
#self.mount_images()
#r = os.system( 'tar -C / -xzf %s' % flashdef.linux_preload_zip ) ### Not needed. fdisk and sfdisk in RAMDISK
self.mount_image_dir()
#
pre_msg = "\n Applying Preload Now !!! -- Do NOT turn system off -- \n"
if not self.blade_test:
self.parent.seq.statusbar.config(text=pre_msg,
bg='yellow',
fg='red')
else:
tst_msg = self.parent.seq.chassis_cmd(
'self.pstatus.cget( "text" )')
pre_msg = tst_msg + "\n\nApply\nPreload\n\n"
self.parent.seq.chassis_cmd('self.pstatus.config( text=%s )' %
repr(pre_msg))
#
if not os.path.exists(flashdef.linux_mops64):
msg = "ERROR:'%s' not found " % flashdef.linux_mops64
raise testexception, msg
else:
self.log.log('Found MOPS64.ZIP')
f = open('/tmp/pre_load.sh', 'w')
xcmd = 'PATH=$PATH:/image:/images:/image/dev:/'
f.write(xcmd + "\n")
xcmd = "echo $PATH"
f.write(xcmd + "\n")
if "preload_tgz" in dir(flashdef) and "hana_preload" in dir(
self.parent.cfgchk):
if "e1350" in dir(flashdef):
self.log.log('Skipping preload on e1350 HANA systems')
return ('skip')
for tgz in flashdef.preload_tgz:
tgz_cmd = 'tar -C / -xzf %s \n' % tgz
f.write(tgz_cmd)
## Create symbolic links to /dev/sda
for dev in ('', '1', '2', '3', '4', '5', '6'):
cmd = 'ln -s /dev/%s%s /dev/HANA%s' % (self.preload_hdd, dev,
dev)
self.log.log(cmd)
self.runcmd(cmd)
if "lvm_grub" in dir(flashdef):
lvm_grub_cmd = 'tar -C / -xzf %s' % flashdef.lvm_grub
f.write(lvm_grub_cmd + '\n')
cpcmd = "unzip -o %s -d /code" % flashdef.linux_mops64
f.write(cpcmd + "\n")
if os.path.exists('AODSTAT.DAT'):
cmd = "mops64.exe /noreboot /AOD=AODSTAT.DAT"
else:
cmd = "mops64.exe /noreboot /AOD=AOD.DAT"
f.write(cmd + "\n")
f.write('echo PRELOAD_RTN=$?\n')
f.close()
#
f = open('/tmp/preload.sh', 'w')
f.write("rm -f /tmp/preload.log\n")
f.write("/tmp/pre_load.sh 2>&1 | tee /tmp/preload.log\n")
f.write("sleep 3\n")
f.close()
#
cmd = "chmod +x /tmp/pre_load.sh"
r = os.system(cmd)
if r != 0:
msg = "ERROR: '%s' cmd failed" % cmd
raise testexception, msg
cmd = "chmod +x /tmp/preload.sh"
r = os.system(cmd)
if r != 0:
msg = "ERROR: '%s' cmd failed" % cmd
raise testexception, msg
#
cmd = "xterm -geometry 90x12+300+5 -e /tmp/preload.sh"
runcmd = popen2.Popen4(cmd)
#
pre_start = time.time()
pre_timeout = pre_start + self.preload_timeout
while time.time() < pre_timeout:
time.sleep(1)
rtn = runcmd.poll()
if rtn != -1:
break
if self.parent != None:
pre_timer = int(time.time() - pre_start)
if not self.blade_test:
stat_msg = pre_msg + ' %d min %02d sec ' % (pre_timer / 60,
pre_timer % 60)
self.parent.seq.statusbar.config(text=stat_msg)
else:
stat_msg = pre_msg + '%d min\n%02d sec' % (pre_timer / 60,
pre_timer % 60)
self.parent.seq.chassis_cmd(
'self.pstatus.config( text=%s )' % repr(stat_msg))
#
cpcmd = "cp /tmp/preload.log ."
r = os.system(cpcmd)
if r != 0:
self.log.log("ERROR %d %s" % (r, cpcmd))
#
if rtn == -1:
msg = "ERROR: Preload TIMEOUT! (%dsec)" % (self.preload_timeout)
os.system('mops32.exe /RE')
os.system('mops32.exe /RE')
raise testexception, msg
#
response = string.strip(runcmd.fromchild.read())
print response
#
f = open('/tmp/preload.log', 'r')
f_log = string.strip(f.read())
f.close()
if string.find(f_log, 'PRELOAD_RTN=0') == -1:
msg = "ERROR: Preload failed!"
msg += "\n See preload.log for MOPS output\n"
pre_rtn = string.find(f_log, 'PRELOAD_RTN')
if pre_rtn != -1:
msg += " /tmp/preload.log = '%s'" % f_log[pre_rtn:]
raise testexception, msg
#
pre_msg = 'PRELOAD completed'
self.log.logp(pre_msg)
#
if self.parent != None:
if self.blade_test:
self.parent.seq.chassis_cmd('self.pstatus.config( text=%s )' %
repr(pre_msg))
else:
self.parent.seq.statusbar.config(text=pre_msg,
bg='white',
fg='black')
#
###########################################################################
##### Audit Boot Setup #########################################
###########################################################################
## Check boottype.dat. if WNTIBOOT is there we need to audit boot.
boot_types = ['WNTIBOOT', 'CANAUDIT', 'CUSTOMIMAGE']
bootfile = "boottype.dat"
if os.path.exists(bootfile):
boot_file = bootfile
elif os.path.exists(bootfile.upper()):
boot_file = bootfile.upper()
else:
msg = 'Could not find %s' % bootfile
raise testexception, msg
f = open(boot_file, 'r')
btype = f.readlines()
f.close()
for line in btype:
boot_type = line.upper().strip()
if boot_type in boot_types:
self.log.log("Found BOOTTYPE = '%s' " % boot_type)
else:
msg = 'Could not determine BOOT TYPE'
raise testexception, msg
if boot_type == 'CUSTOMIMAGE':
if os.path.exists('modules.list.txt'):
open('SAP_PRELOAD', 'w').write("This is a SAP Custom Preload")
self.log.log(
'This is a SAP Custom image that needs to PXE boot VMWare')
## Create PXE1TIME.CFG to PXE Boot VMWare/ EFI1TIME.CFG to Local Boot
for files in ('EFI1TIME.LOG', 'EFI1TIME.CFG', 'PXE1TIME.CFG',
'PXE1TIME.LOG', 'vmware_preload.sh'):
if os.path.exists(files):
os.remove(files)
level_1 = os.environ['L1SERVER']
sut_mtsn = os.environ['MTSN']
vmware_common_dir = flashdef.vmware_common_dir.replace(
'/dfcxact/', '')
pxe1time = []
pxe1time.append('TIMEOUT 8000\n')
pxe1time.append('LABEL VMWARE\n')
pxe1time.append('KERNEL %s/mboot.c32\n' % vmware_common_dir)
pxe1time.append('APPEND -c %s/boot.cfg ' % vmware_common_dir)
pxe1time.append('APPEND MTSN=/dfcxact/mtsn/%s ' % sut_mtsn)
pxe1time.append('L1SERVER=%s PRELOAD=%s ' %
(level_1, self.imgsrvr))
pxe1time.append('ipappend 2\n')
pxe1time.append('DEFAULT VMWARE\n')
efi1time = []
efi1time.append('TIMEOUT 8000\n')
efi1time.append('LABEL VMWARE\n')
efi1time.append(' LOCALBOOT 0\n')
efi1time.append('DEFAULT VMWARE\n')
for line in pxe1time:
open('PXE1TIME.CFG', 'a').write(line)
for line in efi1time:
open('EFI1TIME.CFG', 'a').write(line)
## Create vmware_preload.sh to run from within VMWare
vmw_script = []
if os.path.exists('sapbuild.sh'):
sapbuild_script = 'sapbuild.sh'
else:
sapbuild_script = 'sapbuild'
""" Old DS4 Script
vmw_script.append( 'ftpget -u plclient -p client $L1 sapbuild.sh $MTSN/%s\n' % sapbuild_script )
vmw_script.append( 'ftpget -u plclient -p client $L1 /tmp/modules.list.txt $MTSN/modules.list.txt\n' )
vmw_script.append( 'chmod +x sapbuild.sh\n' )
vmw_script.append( './sapbuild.sh\n' )
vmw_script.append( 'echo PRELOAD_RTN=$? >> mfg_preload.log\n' )
vmw_script.append( 'ftpput -p client -u plclient $L1 $MTSN/vm_preload.log /tmp/vm_preload-*.log\n' )
vmw_script.append( 'ftpput -p client -u plclient $L1 $MTSN/mfg_preload.log mfg_preload.log\n' )
"""
vmw_script.append(
'echo `date "+%d%b%Y %k:%M:%S"` " Copy sapbuild.sh to /tmp">> tester.log\n'
)
vmw_script.append(
'/bin/cp /vmfs/volumes/$MTSN/sapbuild.sh /tmp/sapbuild.sh\n'
)
vmw_script.append(
'echo `date "+%d%b%Y %k:%M:%S"` " Copy modules.list.txt to /tmp" >> tester.log\n'
)
vmw_script.append(
'/bin/cp /vmfs/volumes/$MTSN/modules.list.txt /tmp/modules.list.txt\n'
)
vmw_script.append('chmod +x /tmp/sapbuild.sh\n')
vmw_script.append(
'echo `date "+%d%b%Y %k:%M:%S"` " Running sapbuild.sh" >> tester.log\n'
)
vmw_script.append('/tmp/sapbuild.sh\n')
vmw_script.append('echo PRELOAD_RTN=$? >> mfg_preload.log\n')
vmw_script.append(
'/bin/cp /tmp/vm_preload-*.log /vmfs/volumes/$MTSN/vm_preload.log\n'
)
vmw_script.append(
'echo `date "+%d%b%Y %k:%M:%S"` " sapbuild.sh exited" >> tester.log\n'
)
vmw_script.append('reboot -f\n')
for line in vmw_script:
open('vmware_preload.sh', 'a').write(line)
self.parent.hdd.clear_partition()
os.system("touch PRELOAD.DON")
os.system("sync")
pre_msg = 'Audit Boot Setup DONE'
self.log.logp(pre_msg)
self.log.logp("Create linuxpre.flg")
f = open('linuxpre.flg', 'w')
f.write("preload setup: %s\r\n" % time.asctime())
f.close()
#
if not self.blade_test:
self.parent.seq.statusbar.config(text=pre_msg,
bg='white',
fg='black')
else:
self.parent.seq.chassis_cmd(
'self.pstatus.config( text=%s )' % repr(pre_msg))
self.reboot = 1
return
else:
self.log.log('This is a Custom Image. No Audit Required')
os.system("touch PRELOAD.DON")
os.system("sync")
open('MOPS.PASS', 'w').write("Custom Image Passed")
self.create_cksum()
return
## Set up for Audit Boot
## Create MFGBOOT.BAT
if 'schooner_preload' in dir(self.parent.cfgchk):
# Local Boot 1 TIME both EFI and PXELINUX.CFG
self.runcmd('rm -f PXE1TIME.* EFI1TIME.* ')
f = open('EFI1TIME.CFG', 'w')
f.write('LABEL HDD\n')
f.write(' LOCALBOOT 0 \n')
f.write('DEFAULT HDD\n')
f.close()
self.runcmd('cp EFI1TIME.CFG PXE1TIME.CFG')
elif 'hana_preload' in dir(self.parent.cfgchk):
if "e1350" in dir(flashdef):
self.log.log('Skipping preload on e1350 HANA systems')
return ('skip')
# Local Boot 1 TIME both EFI and PXELINUX.CFG
self.runcmd('rm -f PXE1TIME.* EFI1TIME.* ')
f = open('EFI1TIME.CFG', 'w')
f.write('LABEL HDD\n')
f.write(' LOCALBOOT 0 \n')
f.write('DEFAULT HDD\n')
f.close()
self.runcmd('cp EFI1TIME.CFG PXE1TIME.CFG')
mounts = self.runcmd('mount')
if mounts.lower().find('install1') != -1:
self.runcmd('umount /install1')
if mounts.lower().find('install2') != -1:
self.runcmd('umount /install2')
else:
cmds = []
cmds.append('c: \r\n')
cmds.append('cd \\ibmwork\r\n')
cmds.append('bootsig.com 1234 \r\n')
##cmds.append( 'pause\r\n' )
self.log.log('Create mfgboot.bat')
f = open('/ibm/ibmwork/mfgboot.bat', 'w')
for cmd in cmds:
f.write(cmd)
f.close()
## Copy Files to /ibm/ibmwork
files = ['bootsig.com']
msg = ''
for file in files:
if not os.path.exists(flashdef.prod_dos_path +
file): ## Needs to be updated
msg += '%s is missing\n' % file
self.log.log('%s is missing' % file)
else:
self.log.log('%s found' % file)
os.system(
'cp %s%s /ibm/ibmwork' %
(flashdef.prod_dos_path, file)) ## Needs to be updated
if msg:
raise testexception, msg
## Change Default in PXELINUX.CFG to PRELOAD_LINUX
self.parent.seq.pxelinux_cfg('PRELOAD_LINUX')
os.system("touch PRELOAD.DON")
os.system("sync")
pre_msg = 'Audit Boot Setup DONE'
self.log.logp(pre_msg)
self.log.logp("Create linuxpre.flg")
f = open('linuxpre.flg', 'w')
f.write("preload setup: %s\r\n" % time.asctime())
f.close()
#
if not self.blade_test:
self.parent.seq.statusbar.config(text=pre_msg,
bg='white',
fg='black')
else:
self.parent.seq.chassis_cmd('self.pstatus.config( text=%s )' %
repr(pre_msg))
self.reboot = 1
return
import popen2
pinfo = popen2.Popen4("od /bin/bash")
#print "d: rc: %d" % (rc,)
for line in pinfo.fromchild.readlines():
print line
rc = pinfo.wait()
l = lwbanner
else:
if status != 0:
self.printBannerOrError(banner_or_err)
if self.output_truncated == 1:
status = 1
except OSError,e:
# os error 10 (no child process) is ok
if e.errno ==10:
if verbose : print "No child process %d for wait" % child.pid
else:
raise
else:
if os.name == "posix":
child = popen2.Popen4( sshCommand )
self.child = child
r = child.fromchild
w = child.tochild
w.close()
l = r.readlines()
r.close()
try:
status = child.wait()
banner_or_err = self.readBannerOrError(tmpFd)
tmpFd.close()
os.unlink(tmpBannerFile)
if command:
if status == 255:
self.printBannerOrError(banner_or_err)
}, script)
try:
file(base, 'w').write(baseheader + base_script)
except Exception, err:
print "Can't create base jobscript %s:%s" % (base, err)
sys.exit(1)
if not shared:
## make chkpoint tarball
## options must match pack/unpack from epilogue
tb = os.path.join(chkptdir, tarbfilename)
cmd = "tar -c -p -C %s -f %s . && touch %s.ok" % (chkptdirbase, tb, tb)
try:
p = popen2.Popen4(
cmd
) # execute tar in sub-process, catch both stdout/stderr as stdout
p.tochild.close() # no input to pass
ec = p.wait() # wait for process, catch return value
out = p.fromchild.read() # read output of tar command
except Exception, err:
print "Something went wrong with forking tar (%s): %s" % (cmd, err)
sys.exit(1)
if ec > 0:
print "Tar failed: exitcode %s, output %s, cmd %s" % (ec, out, cmd)
sys.exit(1)
## submit 1 job
submitbase(base, scriptname)
# server initialization
host = 'localhost'
port = 900001 # hopefully this isn't used
backlog = 5
size = 100
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind((host,port))
server.listen(backlog)
# start fireball
os.popen(dir1+'/cleanup.com 2> junk.dat')
os.popen(dir2+'/cleanup.com 2> junk.dat')
os.popen(dir3+'/cleanup.com 2> junk.dat')
os.chdir(dir1)
fire1=popen2.Popen4('./fireball.x > f'+str(timesteps)+'.log')
os.chdir('../'+dir2)
fire2=popen2.Popen4('./fireball.x > f'+str(timesteps)+'.log')
os.chdir('../'+dir3)
fire3=popen2.Popen4('./fireball.x > f'+str(timesteps)+'.log')
os.chdir('..')
# open energy file
f_en=file('energies_'+str(timesteps)+'.txt','w')
def lamFunc(tau):
return 1-tau**5*(70*tau**4-315*tau**3+540*tau**2-420*tau+126)
# begin loop
for i in ind(lam):
print 'lam is',lam[i]
casecount = len(testcases.keys())
testcount = 0
failurecount = 0
for testname in testcases.keys():
ccdef = compilermode["CCDEF"]+testname
if testname[-3:] == "C89":
ccstd = compilermode["C89"]
elif testname[-3:] == "C99":
ccstd = compilermode["C99"]
else:
ccstd = ""
cmd = string.join([cc,ccflags,ccstd,ccdef,inputfilename])
print
print cmd
spawn = popen2.Popen4(cmd)
spawn.wait()
output = spawn.fromchild.readlines()
results = parseResults(output)
if len(testcases[testname])==0:
testcount = testcount + 1 #implicit test for no errors
# Go through the tests of this case and make sure
# the compiler gave a diagnostic
for checkline in testcases[testname].keys():
testcount = testcount + 1
if checkline in results:
if "IGNORE" in testcases[testname][checkline]:
testcount = testcount - 1 #this isn't really a test
# -*- coding: utf-8 -*-
import commands
import popen2
print(commands.getstatusoutput("/bin/echo / | xargs ls"))
print(commands.getoutput("/bin/echo / | xargs ls"))
# This one is safe.
print(commands.getstatus("/bin/echo / | xargs ls"))
print(popen2.popen2("/bin/echo / | xargs ls")[0].read())
print(popen2.popen3("/bin/echo / | xargs ls")[0].read())
print(popen2.popen4("/bin/echo / | xargs ls")[0].read())
print(popen2.Popen3("/bin/echo / | xargs ls").fromchild.read())
print(popen2.Popen4("/bin/echo / | xargs ls").fromchild.read())
def mkdir_remote(host, path):
'''mkdir_remote'''
cmd = "gxpc e -h %s 'mkdir -p %s'" % (host, path)
child = popen2.Popen4(cmd)
status = child.wait()
assert status == 0, (cmd, status)
if childPid != 0:
os.kill(childPid, signal.SIGKILL)
except OSError:
pass
print 'Execution timeout exceeded'
sys.exit(nagiosStateUnknown)
signal.signal(signal.SIGALRM, handleAlarm)
signal.alarm(timeout)
if os.access(qhostPath, os.X_OK) == 0:
print 'Cannot execute %s' % (qhostPath)
sys.exit(nagiosStateUnknown)
qhostPipe = popen2.Popen4('%s -q -h %s' % (qhostPath, hostName))
childPid = qhostPipe.pid
exitStatus = qhostPipe.wait()
childPid = 0
lines = qhostPipe.fromchild.readlines()
if not os.WIFEXITED(
exitStatus) or os.WEXITSTATUS(exitStatus) != 0 or len(lines) < 3:
# qhost didn't exit cleanly
# Check if qhost printed something out
if len(lines) >= 1:
# Print first line of output. Use [:-1] so that we don't get an extra
# carriage return
print 'Error with qhost %s: %s' % ('%s -q -h %s' %
(qhostPath, hostName),
lines[0][:-1])
else:
def executeCommand(command, timeOut = 1200):
"""
_executeCommand_
execute a command, waiting at most timeOut seconds for successful
completation.
Arguments:
command -- the command
timeOut -- the timeout in seconds
Return:
the exit code or -1 if did not finish on time.
"""
startTime = time.time()
# build script file if necessary
if command.find('\n') != -1:
try:
try:
os.remove('script.sh')
except:
pass
aFile = open('script.sh', 'w')
command = '#!/bin/bash\n' + command
aFile.write(command + '\n')
aFile.close()
os.chmod('script.sh', 0o755)
except (IOError, OSError) as msg:
logging.error("Cannot generate execution script: " + str(msg))
return
command = 'bash -c ./script.sh'
# run command
job = popen2.Popen4(command)
output = job.fromchild
# get exit code (if ready)
exitCode = job.poll()
# wait for it to finish
while exitCode == -1:
# check timeout
if (time.time() - startTime) > timeOut:
logging.critical("Timeout exceded for command")
# exceeded, kill the process
try:
os.kill(job.pid, signal.SIGKILL)
# oops, cannot kill it
except OSError:
logging.critical("Cannot kill process")
# abandon execution
return -1
# wait a second
time.sleep(1)
# get exit status
exitCode = job.poll()
try:
out = output.read()
output.close()
# does not work, ignore
except IOError as ie:
logging.info( "Caught an error trying to get output: %s" % str(ie) )
# return exit code
return out, exitCode
size = stat.f_bsize * stat.f_blocks
used = stat.f_bsize * (stat.f_blocks-stat.f_bfree)
except:
size='? '
used='? '
print "%-30s %-70s %15s %15s %-20s" % (mp, share, size, used, fname+':'+str(line+1))
if new_share != "":
print "-> "+new_share
if options.remount:
#import subprocess
#LET OP: subprocess heeft de voorkeur boven popen2, maar werkt niet onder python 2.3 (RH4)
import popen2
print "Reloading autofs"
p=popen2.Popen4(['service','autofs','restart'])
for l in p.fromchild:
print l.replace('\n','')
if p.wait() !=0 :
stderr.write("Could not reload autofs.\n")
for mp in remounts:
print "Umounting %s." % (mp)
p=popen2.Popen4(['umount',mp])
for l in p.fromchild:
print l.replace('\n','')
if p.wait() !=0 :
stderr.write("Could not unmount %s.\n" % (mp))
print "Remounting (mount -a)"
WARNING = value elif key == '-h': usage() exit(0) elif key == '-V': print "Nagios Performance CoPilot client v%.2f" % nagios_pcpclient_version print "Written by Jan-Frode Myklebust <*****@*****.**>" exit(0) if METRIC == 'undefined': usage() exit(3) COMMANDLINE = COMMANDLINE + " " + METRIC if DEBUG: print COMMANDLINE p=popen2.Popen4(COMMANDLINE) exitcode=p.wait() # Get the last line of output from 'pmval': buffer = p.fromchild.readline() while (buffer != ''): output=buffer buffer = p.fromchild.readline() returndata = string.split(output)[0] # Confirm that we have gotten a float, and not # some errormessage in the returndata. If not, # print the error, and give the UNKNOWN exit code:
print "TEST ^C to mpiexec"
PYEXT = '.py'
NMPDS = 1
HFILE = 'temph'
import os, socket
from mpdlib import MPDTest
mpdtest = MPDTest()
os.environ['MPD_CON_EXT'] = 'testing'
os.system("mpdallexit%s 1> /dev/null 2> /dev/null" % (PYEXT))
temph = open(HFILE, 'w')
for host in clusterHosts:
print >> temph, host
temph.close()
os.system("mpdboot%s -f %s -n %d" % (PYEXT, HFILE, NMPDS))
import popen2
runner = popen2.Popen4("mpiexec%s -n 2 infloop -p" %
(PYEXT)) # -p => don't print
import time ## give the mpiexec
time.sleep(2) ## time to get going
os.system("kill -INT %d" % (runner.pid)) # simulate user ^C
expout = ''
rv = mpdtest.run(cmd="mpdlistjobs%s #2" % (PYEXT), chkOut=1, expOut=expout)
os.system("mpdallexit%s 1> /dev/null 2> /dev/null" % (PYEXT))
os.system("killall -q infloop") ## just to be safe
# test:
print "TEST re-knit a ring"
PYEXT = '.py'
NMPDS = 3
HFILE = 'temph'
import os, socket
from mpdlib import MPDTest
def echo(word):
cmd = 'echo ' + word
subproc = popen2.Popen4(cmd)
out = subproc.fromchild.readlines() # SExtractor output
out = out[0][:-1] # LIST OF 1 STRING WITH \n AT END
return out
def _have_bin(name):
"""
Determines whether a particular binary is available on the host system.
"""
return popen2.Popen4('which %s' % name).wait() == 0
def _run(self, args):
cmd = "/sbin/dhcpcd " + args
a = popen2.Popen4(cmd)
return a.wait()
def run_rexport(self, c, j, a):
self.job = j
#print "set job to",self.job,j
j.status = Status.running
j.output = []
self.pobj = popen2.Popen4(c)
while self.pobj.poll() < 0:
line = "dummy"
while line != "":
try:
line = self.pobj.fromchild.readline()
m = self.re1.findall(line)
if len(m) > 0:
self.job.node = m[0]
#print "thread read",line
if self.re2.match(line):
j.output.append(line)
#else:
# print "no match line:",line
except:
line = ""
# sleep (this thread only)
time.sleep(5) # maybe make this configurable ?
if self.pobj.poll() != 9:
j.status = Status.completed
lines = self.pobj.fromchild.readlines(
) # this will hang for killed processes??
for l in lines:
time.sleep(0.05)
j.output.append(l)
else:
j.status = Status.failed
j.output.append("No more output: process was killed")
self.pobj.tochild.close()
self.pobj.fromchild.close()
j.returnValue = self.pobj.poll()
# now grab any interesting info from the output
if gmtkPT.regex.is_gmtkTime.match(c):
m = gmtkPT.regex.partitions.findall(''.join(j.output))
if len(m) > 0:
j.partitions = int(m[0])
elif gmtkPT.regex.is_gmtkTriangulate_find_boundary.match(c):
j.sum_jtWeight = -1 # not keeping jt weight of existing trianguklation
elif gmtkPT.regex.is_gmtkTriangulate.match(c):
m = gmtkPT.regex.jtWeights.findall(' '.join(j.output))
if len(m) > 0:
j.jtWeights = ' , '.join(m)
j.sum_jtWeight = float(0.0)
for mi in m:
j.sum_jtWeight = j.sum_jtWeight + float(mi)
# delete popen4 object
self.pobj = None
# and save this info in a file
self.job.save_stats(append=False)
def render_macro(self, req, name, content):
# args will be null if the macro is called without parenthesis.
if not content:
return ''
# parse arguments
# we expect the 1st argument to be a filename (filespec)
args = content.split(',')
if len(args) == 0:
raise Exception("No argument.")
filespec = args[0]
size_re = re.compile('[0-9]+%?$')
attr_re = re.compile('(align|border|width|height|alt'
'|title|longdesc|class|id|usemap)=(.+)')
quoted_re = re.compile("(?:[\"'])(.*)(?:[\"'])$")
attr = {}
style = {}
nolink = False
for arg in args[1:]:
arg = arg.strip()
if size_re.match(arg):
# 'width' keyword
attr['width'] = arg
continue
if arg == 'nolink':
nolink = True
continue
if arg in ('left', 'right', 'top', 'bottom'):
style['float'] = arg
continue
match = attr_re.match(arg)
if match:
key, val = match.groups()
m = quoted_re.search(val) # unquote "..." and '...'
if m:
val = m.group(1)
if key == 'align':
style['float'] = val
elif key == 'border':
style['border'] = ' %dpx solid' % int(val);
else:
attr[str(key)] = val # will be used as a __call__ keyword
# parse filespec argument to get module and id if contained.
parts = filespec.split(':')
url = None
if len(parts) == 3: # module:id:attachment
if parts[0] in ['wiki', 'ticket']:
module, id, file = parts
else:
raise Exception("%s module can't have attachments" % parts[0])
elif len(parts) == 2:
from trac.versioncontrol.web_ui import BrowserModule
try:
browser_links = [link for link,_ in
BrowserModule(self.env).get_link_resolvers()]
except Exception:
browser_links = []
if parts[0] in browser_links: # source:path
module, file = parts
rev = None
if '@' in file:
file, rev = file.split('@')
url = req.href.browser(file, rev=rev)
raw_url = req.href.browser(file, rev=rev, format='raw')
desc = filespec
else: # #ticket:attachment or WikiPage:attachment
# FIXME: do something generic about shorthand forms...
id, file = parts
if id and id[0] == '#':
module = 'ticket'
id = id[1:]
elif id == 'htdocs':
raw_url = url = req.href.chrome('site', file)
desc = os.path.basename(file)
elif id in ('http', 'https', 'ftp'): # external URLs
raw_url = url = desc = id+':'+file
else:
module = 'wiki'
elif len(parts) == 1: # attachment
# determine current object
# FIXME: should be retrieved from the formatter...
# ...and the formatter should be provided to the macro
file = filespec
module, id = 'wiki', 'WikiStart'
path_info = req.path_info.split('/',2)
if len(path_info) > 1:
module = path_info[1]
if len(path_info) > 2:
id = path_info[2]
if module not in ['wiki', 'ticket']:
raise Exception('Cannot reference local attachment from here')
else:
raise Exception('No filespec given')
if not url: # this is an attachment
from trac.attachment import Attachment
attachment = Attachment(self.env, module, id, file)
url = attachment.href(req)
dia_url = attachment.href(req, format='raw')
dia_path = attachment.path
dia_filename = attachment.filename
img_url = dia_url.replace(".dia",".png")
img_path = dia_path.replace('.dia','.png')
img_filename = dia_filename.replace('.dia','.png')
self.env.log.info('Getting file modification times.')
#get file modification times
try:
dia_mtime = os.path.getmtime(dia_path)
except Exception:
raise Exception('File does not exist: %s', dia_path)
try:
img_mtime = os.path.getmtime(img_path)
except Exception:
img_mtime = 0
self.env.log.info('Comparing dia and png file modification times : %s, %s',dia_mtime,img_mtime)
# if diagram is newer than image, then regenerate image
if (dia_mtime > img_mtime):
try:
# TODO: read this comment and adjust the command line to taste
# You should probably use the correct path
# The options are correct for the 0.96.1, but you may bee to adjust them
diacmd = 'dia -l --filter=png --export='+img_path+' '+dia_path
self.env.log.info('Running Dia : %s',diacmd)
f = popen2.Popen4(diacmd)
lines = []
while (f.poll() == -1):
lines += f.fromchild.readlines()
f.wait()
#ecode = os.spawnl(os.P_WAIT,'/usr/bin/dia','/usr/bin/dia','-l','--export-to-format=png', '--export='+img_path, dia_path)
self.env.log.info('Exiting Dia')
except Exception, e:
self.env.log.info('Dia failed with exception= %s',e)
raise Exception('Dia execution failed.')
try:
attachment._fetch(img_filename)
except Exception:
db = self.env.get_db_cnx()
handle_ta = True
attachment.size = 0
attachment.time = 0
cursor = db.cursor()
cursor.execute("INSERT INTO attachment "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s)",
(attachment.parent_type, attachment.parent_id, img_filename,
attachment.size, attachment.time, 'PNG render of a DIA file', attachment.author,
attachment.ipnr))
attachment.filename = img_filename
self.env.log.info('New attachment: %s', img_filename)
if handle_ta:
db.commit()
desc = 'JPG render of a DIA file.'
import glob
import os
import popen2
import sys
ok = []
new = []
fail = []
fault = []
for m in glob.glob('[A-Za-z0-9]*.l1'):
print m,
sys.stdout.flush()
cmd = '../l1 -6 -b -w ' + m
p = popen2.Popen4(cmd)
rv = p.wait()
if os.WIFSIGNALED(rv):
print 'fault'
fault.append(m)
continue
out = p.fromchild.read()
del p
fn = m + '.out'
if os.path.exists(fn):
f = open(fn)
s = f.read()
f.close()
if s == out:
print 'ok'
ok.append(m)
def popen(server, cmd):
global p
if server != 'localhost':
cmd = 'ssh -f %s "%s"' % (server, cmd)
p = popen2.Popen4(cmd)
export OUTDIR=%s
export ESEXE="$ESTOOLKIT/ESBuilder -output $OUTDIR -time %s -grade %s -view truncated -dataVersionName %s -db %s -logFile stdout"
echo "Executing: $ESEXE"
$ESEXE -add %s
"""%(path,oDir,tStamp,grade,svName,DB,iFile)
print script
# write out and execute script
fileName = "trunc_run%s_%s.sh"%(run,int(time.mktime(time.localtime())))
oFile = open(fileName,'w')
oFile.write(script)
oFile.close()
res = popen2.Popen4("sh %s"%fileName)
res.wait()
result= res.fromchild.read()
print result
os.remove(fileName)
# start transaction
cu.execute('BEGIN')
# find out key file id of original keyFile
query = "SELECT DISTINCT keyFileId from KeyFile,Version WHERE run='%s' and view='all' and grade='%s' and Version.graphid=KeyFile.graphid;"%(run,grade)
#query = "SELECT keyFileId from KeyFile WHERE run='%s' and view='all'"%run
cu.execute(query)
tup = cu.fetchone()
try:
def run(self):
boto.log.info('running:%s' % self.command)
self.process = popen2.Popen4(self.command)
if (self.wait):
return self.getStatus()
#!/usr/bin/env python
"""Setup script for the mmpython distribution."""
__revision__ = "$Id: setup.py,v 1.12 2004/05/25 14:10:19 the_krow Exp $"
from distutils.core import setup, Extension
import popen2
import version
extensions = [ Extension('mmpython/disc/cdrom', ['disc/cdrommodule.c']) ]
# check for libdvdread (bad hack!)
# Windows does not have Popen4, so catch exception here
try:
child = popen2.Popen4('gcc -ldvdread')
if child.fromchild.readline().find('cannot find') == -1:
# gcc failed, but not with 'cannot find', so libdvd must be
# somewhere (I hope)
extensions.append(Extension('mmpython/disc/ifoparser', ['disc/ifomodule.c'],
libraries=[ 'dvdread' ],
library_dirs=['/usr/local/lib'],
include_dirs=['/usr/local/include']))
child.wait()
except AttributeError, e:
print "No Popen4 found. This seems to be Windows."
print "Installing without libdvdread support."
# Hack: disable extensions for Windows.
# This would better be done by a clean detect of windows. But how?
extensions = []
#!/usr/bin/env python
import glob
import os
import popen2
import sys
import platform
import time
os.putenv('GIT_PAGER', 'cat')
p = popen2.Popen4('git show --pretty=oneline')
rv = p.wait()
if rv == 0:
commit = p.fromchild.readlines()[0].rstrip()
else:
commit = 'commit unknown'
now = time.strftime("%Y.%2m.%2d.%2H.%2M.%2S", time.localtime())
uname = os.uname()
fn = 'perfdat/' + now
f = open(fn, 'w')
print >> f, commit
print >> f, now
print >> f, uname[0]
print >> f, uname[1]
print >> f, uname[2]
print >> f, uname[3]
print >> f, uname[4]
fault = []
def manage_createBackup(self,
includeEvents=None,
includeMysqlLogin=None,
timeout=120,
REQUEST=None,
writeMethod=None):
"""
Create a new backup file using zenbackup and the options specified
in the request.
This method makes use of the fact that DataRoot is a Commandable
in order to use Commandable.write
"""
import popen2
import fcntl
import time
import select
def write(s):
if writeMethod:
writeMethod(s)
elif REQUEST:
self.write(REQUEST.RESPONSE, s)
footer = None
if REQUEST and not writeMethod:
header, footer = self.commandOutputTemplate().split('OUTPUT_TOKEN')
REQUEST.RESPONSE.write(str(header))
write('')
try:
cmd = binPath('zenbackup') + ' -v10'
if not includeEvents:
cmd += ' --no-eventsdb'
if not includeMysqlLogin:
cmd += ' --no-save-mysql-access'
try:
timeout = int(timeout)
except ValueError:
timeout = 120
timeout = max(timeout, 1)
child = popen2.Popen4(cmd)
flags = fcntl.fcntl(child.fromchild, fcntl.F_GETFL)
fcntl.fcntl(child.fromchild, fcntl.F_SETFL, flags | os.O_NDELAY)
endtime = time.time() + timeout
write('%s' % cmd)
write('')
pollPeriod = 1
firstPass = True
while time.time() < endtime and (firstPass or child.poll() == -1):
firstPass = False
r, w, e = select.select([child.fromchild], [], [], pollPeriod)
if r:
t = child.fromchild.read()
# We are sometimes getting to this point without any data
# from child.fromchild. I don't think that should happen
# but the conditional below seems to be necessary.
if t:
write(t)
if child.poll() == -1:
write('Backup timed out after %s seconds.' % timeout)
import signal
os.kill(child.pid, signal.SIGKILL)
write('DONE')
except Exception:
write('Exception while performing backup.')
write('type: %s value: %s' % tuple(sys.exc_info()[:2]))
else:
if REQUEST or writeMethod:
audit('UI.Backup.Create')
write('')
if REQUEST and footer:
REQUEST.RESPONSE.write(footer)
def mkMsg(self):
"""Create and write module level message for this class.
Most of this is just compiling the info. meta is a dictionary
of lists where each list is a list of tuples describing the
tag lines for the particular section of the message. This tuple
format conforms to that used by the xmlMessage class which is
modeled on basic python argument passing, i.e. (key,*value,**attr).
"""
# getting the version of project_coords
project_coords_cmd = 'project_coords --version'
outp = popen2.Popen4(project_coords_cmd)
outpline = outp.fromchild.readlines()
pcoorVer = outpline[0].split()[-1]
self.meta = {}
self.meta['module'] = []
self.meta['meta'] = []
self.meta['input'] = []
self.meta['output'] = []
self.meta['errorlist'] = []
self.meta['module'].append(
('module', 'name=' + self.modName, 'version=' + __version__,
'dataset=' + self.obsName))
self.meta['module'].append(('root', self.root))
self.meta['meta'].append(('meta', ))
self.meta['meta'].append(('depend', ))
self.meta['meta'].append(('pkg', ))
self.meta['meta'].append(('name', 'python'))
self.meta['meta'].append(('version', pyversion.split()[0]))
self.meta['meta'].append(('pkg', ))
self.meta['meta'].append(('name', 'pyfits'))
self.meta['meta'].append(('version', pyfits.__version__.split()[0]))
self.meta['meta'].append(('pkg', ))
self.meta['meta'].append(('name', 'project_coords'))
self.meta['meta'].append(('version', pcoorVer))
self.meta['meta'].append(('pkg', ))
self.meta['meta'].append(('name', 'Guide Star Catalog'))
self.meta['meta'].append(
('version', _URL_.split("/")[-1].split("q")[0]))
# SExtractor info
sub = subprocess.Popen(['sex', '--version'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
close_fds=True)
outp = sub.stdout.readlines()
name = outp[0].split()[0]
ver = outp[0].split()[2]
self.meta['meta'].append(('pkg', ))
self.meta['meta'].append(('name', name))
self.meta['meta'].append(('version', ver))
cmdline1 = 'sex fitsfile -c self.InParFileName'
self.meta['meta'].append(('commandline', cmdline1))
del outp, sub, name, ver
if self.errorList:
self.meta['errorlist'].append(('errorlist', ))
for pkg, err in self.errorList:
self.meta['errorlist'].append(
('erroritem', err, 'frompkg=' + pkg))
# input section
self.meta['input'].append(('input', ))
for f in self.inputList:
if string.find(f, "_asn") == -1:
self.meta['input'].append(('file', 'type=image/x-fits'))
self.meta['input'].append(('name', os.path.join("Images", f)))
else:
self.meta['input'].append(('file', 'type=image/x-fits'))
self.meta['input'].append(('name', os.path.join("Images", f)))
# output section
if self.outputList:
self.meta['output'].append(('output', ))
for f in self.outputList.keys():
if string.find(f, ".xml") == -1:
self.meta['output'].append(('file', 'type=image/x-fits'))
self.meta['output'].append(('name', os.path.join("Images", f)))
for pred in self.outputList[f]:
self.meta['output'].append(
('predecessor', os.path.join("Images", pred)))
else:
self.meta['output'].append(('file', 'type=text/xml'))
self.meta['output'].append(('name', os.path.join("Images", f)))
for pred in self.outputList[f]:
self.meta['output'].append(
('predecessor', os.path.join("Images", pred)))
# pass this dictionary to the class pMessage...
msgFile = os.path.join(self.messagedir, self.modName + "_module.xml")
mmsg = pMessage(self.meta)
mmsg.writeMsg(msgFile)
return