def __init__(self, console, history_file=default_history_file):
Runtime.getRuntime().addShutdownHook(Thread(self))
self.history_file = history_file
self.history = []
self.loadHistory()
self.console = console
self.index = len(self.history) - 1
self.last = ""
def __init__(self, console, history_file=default_history_file):
Runtime.getRuntime().addShutdownHook(Thread(self))
self.history_file = history_file
self.history = []
self.loadHistory()
self.console = console
self.index = len(self.history) - 1
self.last = ""
def run(self, options, args):
if options.supported or options.supported_html:
return storytext.guishared.ScriptEngine.run(self, options, args)
class ShutdownHook(Thread):
def run(tself):#@NoSelf
self.cleanup(options.interface)
if not options.disable_usecase_names:
hook = ShutdownHook()
Runtime.getRuntime().addShutdownHook(hook)
return storytext.scriptengine.ScriptEngine.run(self, options, args)
def actionPerformed(self, event):
browsers = ["google-chrome", "firefox", "opera", "epiphany", "konqueror", "conkeror", "midori", "kazehakase", "mozilla"]
osName = System.getProperty("os.name")
helpHTML = ClassLoader.getSystemResource("help.html").toString()
if osName.find("Mac OS") == 0:
Class.forName("com.apple.eio.FileManager").getDeclaredMethod( "openURL", [String().getClass()]).invoke(None, [helpHTML])
elif osName.find("Windows") == 0:
Runtime.getRuntime().exec( "rundll32 url.dll,FileProtocolHandler " + helpHTML)
else:
browser = None
for b in browsers:
if browser == None and Runtime.getRuntime().exec(["which", b]).getInputStream().read() != -1:
browser = b
Runtime.getRuntime().exec([browser, helpHTML])
def browseURI(uri):
osName = System.getProperty("os.name")
rt = Runtime.getRuntime()
if osName.startswith("Mac OS"):
rt.exec('open "%s"' % uri)
else:
if osName.startswith("Windows"):
ProcessBuilder(["cmd", "/C", "start", uri]).start()
else:
browsers = ["google-chrome", "firefox", "opera", "konqueror", "epiphany", "mozilla", "netscape"]
for b in browsers:
exists = rt.exec("which %s" % b).getInputStream().read()
if exists != -1:
Runtime.getRuntime().exec("%s %s" % (b, uri))
return
def launchProgramNoWait(args, workingDir = None): if workingDir != None and not isinstance(workingDir, File): workingDir = File(workingDir) process = Runtime.getRuntime().exec(args, None, workingDir) OutputThread(process.getInputStream(), System.out).start() OutputThread(process.getErrorStream(), System.err).start() return process
def launchProgramNoWait(args, workingDir = None): if workingDir != None and not isinstance(workingDir, File): workingDir = File(workingDir) process = Runtime.getRuntime().exec(args, None, workingDir) OutputThread(process.getInputStream(), System.out).start() OutputThread(process.getErrorStream(), System.err).start() return process
def syslog( level, message ):
try:
ps = Runtime.getRuntime().exec(['logger', '-p', '%s.%s' % (LogConfig.syslogFacility, level), '-t', LogConfig.syslogTag, message])
return ps.waitFor()
except:
log(INFO_, level + " " + message)
return 0
def system(cmd): """ system(cmd): executes cmd in a shell Jpython currently lacks a system command in its os module. This is a temporary filler in till a better one comes along the way """ r= Runtime.getRuntime() try: p = r.exec(cmd) p.waitFor() except: raise 'Error executing shell command: ' + cmd lnr_err = LineNumberReader(InputStreamReader(p.getErrorStream())) err_lines = [] while 1: line_err = lnr_err.readLine() if not line_err: break else: print line_err # lnr = LineNumberReader(InputStreamReader(p.getInputStream())) #lines = [] while 1: line = lnr.readLine() if not line: break else: print line
def cpu_count():
# Python 2.6+
try:
import multiprocessing
return multiprocessing.cpu_count()
except (ImportError, NotImplementedError):
pass
# POSIX
try:
import os
res = int(os.sysconf('SC_NPROCESSORS_ONLN'))
if res > 0:
return res
except (AttributeError, ValueError):
pass
# Windows
try:
res = int(os.environ['NUMBER_OF_PROCESSORS'])
if res > 0:
return res
except (KeyError, ValueError):
pass
# jython
try:
from java.lang import Runtime
runtime = Runtime.getRuntime()
res = runtime.availableProcessors()
if res > 0:
return res
except ImportError:
pass
# BSD
try:
import subprocess
sysctl = subprocess.Popen(['sysctl', '-n', 'hw.ncpu'],
stdout=subprocess.PIPE)
scStdout = sysctl.communicate()[0]
res = int(scStdout)
if res > 0:
return res
except (OSError, ValueError):
pass
# Linux
try:
res = open('/proc/cpuinfo').read().count('processor\t:')
if res > 0:
return res
except IOError:
pass
return 0
def cachedBlast(fileName, blastLocation, database, eValue, query, pipeline, force = False):
"""
Performs a blast search using the blastp executable and database in blastLocation on
the query with the eValue. The result is an XML file saved to fileName. If fileName
already exists the search is skipped. If remote is true then the search is done remotely.
"""
if not os.path.isfile(fileName) or force:
output = open(fileName, "w")
command = [blastLocation + "/bin/blastp",
"-evalue", str(eValue),
"-outfmt", "5",
"-query", query,
"-num_threads", str(Runtime.getRuntime().availableProcessors()),
"-db", database]
blastProcess = subprocess.Popen(command,
stdout = output)
while blastProcess.poll() == None:
if pipeline.exception:
print "Stopping in blast"
psProcess = subprocess.Popen(["ps", "aux"], stdout = subprocess.PIPE)
awkProcess = subprocess.Popen(["awk", "/" + " ".join(command).replace("/", "\\/") + "/"], stdin = psProcess.stdout, stdout = subprocess.PIPE)
for line in awkProcess.stdout:
subprocess.Popen(["kill", "-9", re.split(r"\s+", line)[1]])
output.close()
raise pipeline.exception
if blastProcess.poll() != 0:
raise OSError()
output.close()
try:
return parseBlast(fileName)
except SAXParseException:
print 'Retry'
return cachedBlast(fileName, blastLocation, database, eValue, query, pipeline, True)
def system(cmd): """ system(cmd): executes cmd in a shell Jpython currently lacks a system command in its os module. This is a temporary filler in till a better one comes along the way """ r= Runtime.getRuntime() try: p = r.exec(cmd) p.waitFor() except: raise 'Error executing shell command: ' + cmd lnr_err = LineNumberReader(InputStreamReader(p.getErrorStream())) err_lines = [] while 1: line_err = lnr_err.readLine() if not line_err: break else: print line_err # lnr = LineNumberReader(InputStreamReader(p.getInputStream())) #lines = [] while 1: line = lnr.readLine() if not line: break else: print line
def cpu_count():
"""Returns the number of cpus"""
# jython
try:
from java.lang import Runtime
runtime = Runtime.getRuntime()
res = runtime.availableProcessors()
if res > 0:
return res
except ImportError:
pass
# POSIX
try:
res = int(os.sysconf('SC_NPROCESSORS_ONLN'))
if res > 0:
return res
except (AttributeError,ValueError):
pass
# Windows
try:
res = int(os.environ['NUMBER_OF_PROCESSORS'])
if res > 0:
return res
except (KeyError, ValueError):
pass
#Could not get the number of cpus. Return 1
return 1
def start_com(self):
s_is = Scanner(self.s.getInputStream())
file_name = s_is.nextLine()
print(file_name)
pro = Runtime.getRuntime().exec(file_name)
try:
pro_os = PrintStream(pro.getOutputStream())
except:
pass
print("reached")
try:
f = open(raw_input("Enter input file name: "), "r")
lp = []
print(str(f) + " file")
for k in f:
lp.append(str(k))
for k in lp:
pro_os.print(k)
except:
pass
finally:
pro_os.close()
pro_is = Scanner(pro.getInputStream())
s_os = PrintStream(self.s.getOutputStream())
s_os.println(file_name + " recieved")
while pro_is.hasNext():
s_os.println(pro_is.nextLine())
s_is.close()
s_os.close()
self.s.close()
def init(self, config):
#self._numWorkers = 1
self._numWorkers = Runtime.getRuntime().availableProcessors()
self._maximaPath = config["maximaPath"]
self._threadPool = ThreadPool(self._numWorkers)
self._maximaPool = Queue.Queue(self._numWorkers)
for i in range(self._numWorkers):
self.launchMaximaInstance()
def get_cpu_count():
if os.name == 'java':
from java.lang import Runtime
runtime = Runtime.getRuntime()
cpu_count = runtime.availableProcessors()
else:
import multiprocessing
cpu_count = multiprocessing.cpu_count()
return cpu_count
def main():
global myarg0
try:
myarg0
except NameError:
print "Please input the class you want to open source file"
return
path = "/home/LiGeng/Desktop/Javasrc"
s = myarg0.split(".")
for x in s:
path = ("%s/%s")%(path, x)
path = ("%s.java") % path
print path
Runtime.getRuntime().exec(("emacs %s")%path)
def run(string, args=[], callback=None, callbackOnErr=False):
def out (exit, call, inp, err):
return {
"exitCode": exit,
"callbackReturn": call,
"inputArray": inp,
"errorArray": err
}
tmp = File.createTempFile('tmp', None)
tmp.setExecutable(True)
writer = FileOutputStream(tmp);
writer.write(string)
writer.flush()
writer.close()
try:
process = Runtime.getRuntime().exec([tmp.getAbsolutePath()] + ([str(i) for i in args] or []))
process.waitFor()
inp = BufferedReader(InputStreamReader(process.getInputStream()))
err = BufferedReader(InputStreamReader(process.getErrorStream()))
errFlag = False
inputArray = []
errorArray = []
holder = inp.readLine()
while holder != None:
print holder
inputArray += [holder]
holder = inp.readLine()
holder = err.readLine()
while holder != None:
errFlag = True
errorArray += [holder]
holder = err.readLine()
tmp.delete()
if errFlag:
if callback and callbackOnErr: return out(1, callback(out(1, None, inputArray, errorArray)), inputArray, errorArray)
else: return out(1, None, inputArray, errorArray)
else:
if callback: return out(0, callback(out(0, None, inputArray, [])), inputArray, [])
else: return out(0, None, inputArray, [])
except Exception as e:
print str(e)
tmp.delete()
if callback and callbackOnErr: return out(3, callback(out(3, None, [], str(e).split("\n"))), [], str(e).split("\n"))
else: return out(3, None, [], str(e).split("\n"))
def play(self): num_threads = Runtime.getRuntime().availableProcessors() executor = Executors.newFixedThreadPool(num_threads) callables = [_Worker(start_pos) for start_pos in self.positions] futures = executor.invokeAll(callables) # calculate stats for future in futures: worker = future.get() self.process_scores(worker) executor.shutdown()
def executeCommand(comm, params):
parameters = " ".join(params)
command = comm + " " + parameters
if isPython:
os.system(command)
else:
from java.lang import Runtime
runner = Runtime.getRuntime()
getattr(runner, "exec")(command) # runner.exec is a Python syntax error
return True
def get_cpu_count():
cpu_count = 1
if os.name == 'java':
from java.lang import Runtime
runtime = Runtime.getRuntime()
cpu_count = runtime.availableProcessors()
else:
import multiprocessing
cpu_count = multiprocessing.cpu_count()
return cpu_count
def Exec():
from java.lang import Runtime
from java.io import BufferedReader
from java.io import InputStreamReader
process = Runtime.getRuntime().exec(cmd);
inp = BufferedReader(InputStreamReader(process.getInputStream(),"euc-kr"))
out = ""
line = inp.readLine()
while line:
out = out + line
line = inp.readLine()
def getCalibPG(instr, type):
cmd = "findcalib -i"+instr+" --listall " + type
print cmd
from java.lang import Runtime
proc = Runtime.getRuntime().exec(cmd)
print proc
import ExtTools.monq.stuff.Exec
thing = ExtTools.monq.stuff.Exec(proc)
if not thing.done():
pass
print thing.getOutputText()
return None # should make a system call to findcalib
def newFixedThreadPool(n_threads=0, name="jython-worker"):
""" Return an ExecutorService whose Thread instances belong
to the same group as the caller's Thread, and therefore will
be interrupted when the caller is.
n_threads: number of threads to use.
If zero, use as many as available CPUs.
If negative, use as many as available CPUs minus that number,
but at least one. """
if n_threads <= 0:
n_threads = max(1,
Runtime.getRuntime().availableProcessors() + n_threads)
return Executors.newFixedThreadPool(n_threads,
ThreadFactorySameGroup(name))
def barcodes(path,filename):
max_dimension = 3;
resolution = 100;
num_landmark_points = 50;
nu = 1;
num_divisions = 100;
Dintervals = [];
cloud = [(math.cos(2*x*math.pi/resolution)+random.uniform(-.2,.2),
math.sin(2*x*math.pi/resolution)+random.uniform(-.2,.2)) for x in xrange(100)];
print "Creating maxmin landmark selector"
print "Total available memory: " + str(Runtime.getRuntime().totalMemory())
# create a sequential maxmin landmark selector
landmark_selector = api.Plex4.createMaxMinSelector(cloud, num_landmark_points);
#R = landmark_selector.getMaxDistanceFromPointsToLandmarks()
max_filtration_value = 1.2 # R*.7;
print "Constructing lazy witness stream"
# create a lazy witness stream
stream = streams.impl.LazyWitnessStream(landmark_selector.getUnderlyingMetricSpace(),
landmark_selector,
max_dimension,
max_filtration_value,
nu,
num_divisions);
stream.finalizeStream()
print "Lazy witness stream construction complete"
# print out the size of the stream
#num_simplices = stream.getSize()
# get persistence algorithm over Z/2Z
persistence = api.Plex4.getModularSimplicialAlgorithm(max_dimension, 2);
print "Computing intervals"
# compute the intervals
intervals = persistence.computeIntervals(stream);
# process and pickle the intervals
for dimension in xrange(max_dimension):
temp = [] # collection of intervals at dimension
temp0 = intervals.getIntervalsAtDimension(dimension)
for ninterval in xrange(len(temp0)):
temp.append([temp0[ninterval].getStart(),
temp0[ninterval].getEnd()]) # intervals as pairs [start,end]
Dintervals.append([dimension, temp]) # adds dimension information for the collection of intervals
#====================================================================
# Dintervals are in the format:
# [[dim_1,listofintervalsofdim_1],...,[dim_k,listofintervalsofdim_k]]
#====================================================================
cp.dump(Dintervals, open(path + filename, 'wb'))
cp.dump(cloud,open(path + filename +"cloud",'wb'))
print "Intervals saved."
def get_jython_free_mem(self):
# Force garbage collection first.
from java.lang import Runtime
from java.lang.ref import WeakReference
from java.lang import Object
from java.lang import System
obj = Object()
ref = WeakReference(obj)
obj = None
while ref.get() is not None:
System.gc()
# Calculate approx. available memory.
return Runtime.getRuntime().freeMemory()
def execute(self, cmd):
"""Execute cmd in a shell, and return the java.lang.Process instance.
Accepts either a string command to be executed in a shell,
or a sequence of [executable, args...].
"""
shellCmd = self._formatCmd(cmd)
env = self._formatEnvironment(self.environment)
try:
p = Runtime.getRuntime().exec(shellCmd, env, File(os.getcwd()))
return p
except IOException as ex:
raise OSError(
0, "Failed to execute command (%s): %s" % (shellCmd, ex))
def execute(self, cmd):
"""Execute cmd in a shell, and return the java.lang.Process instance.
Accepts either a string command to be executed in a shell,
or a sequence of [executable, args...].
"""
shellCmd = self._formatCmd(cmd)
env = self._formatEnvironment(self.environment)
try:
p = Runtime.getRuntime(). exec (shellCmd, env, File(os.getcwd()))
return p
except IOException, ex:
raise OSError(
0, "Failed to execute command (%s): %s" % (shellCmd, ex))
def get_cpu_count(conf):
""" Return cpu count.
Arguments:
run id: The run id
conf: configuration dictionary
"""
# Get the number of cpu
cpu_count = int(conf[BCL2FASTQ_THREADS_KEY])
if cpu_count < 1:
cpu_count = Runtime.getRuntime().availableProcessors()
return cpu_count
def run(self, args, javaHome, jythonHome, jythonOpts, internals=False):
''' creates a start script, executes it and captures the output '''
(starterPath, outfilePath) = self.writeStarter(args, javaHome, jythonHome, jythonOpts, internals)
try:
process = Runtime.getRuntime().exec(starterPath)
stdoutMonitor = StdoutMonitor(process)
stderrMonitor = StderrMonitor(process)
stdoutMonitor.start()
stderrMonitor.start()
while self.isAlive(process):
Thread.sleep(300)
return self.getOutput(outfilePath)
finally:
os.remove(starterPath)
os.remove(outfilePath)
def run_command(args_list):
output = ""
error = ""
p = Runtime.getRuntime().exec(args_list)
i = p.waitFor()
stdOutput = BufferedReader(InputStreamReader(p.getInputStream()))
returnedValues = stdOutput.readLine()
while returnedValues != None:
output += returnedValues + "\n"
returnedValues = stdOutput.readLine()
stdError = BufferedReader(InputStreamReader(p.getErrorStream()))
returnedValues = stdError.readLine()
while returnedValues != None:
error += returnedValues + "\n"
returnedValues = stdError.readLine()
return (i, output, error)
def getShiftFromViews(v1, v2):
# Thread pool
exe = Executors.newFixedThreadPool(
Runtime.getRuntime().availableProcessors())
try:
# PCM: phase correlation matrix
pcm = PhaseCorrelation2.calculatePCM(
v1, v2, ArrayImgFactory(FloatType()), FloatType(),
ArrayImgFactory(ComplexFloatType()), ComplexFloatType(), exe)
# Minimum image overlap to consider, in pixels
minOverlap = v1.dimension(0) / 10
# Returns an instance of PhaseCorrelationPeak2
peak = PhaseCorrelation2.getShift(pcm, v1, v2, nHighestPeaks,
minOverlap, True, True, exe)
except Exception, e:
print e
def run(self, args, javaHome, jythonHome, jythonOpts):
''' creates a start script, executes it and captures the output '''
(starterPath, outfilePath) = self.writeStarter(args, javaHome,
jythonHome, jythonOpts)
try:
process = Runtime.getRuntime().exec(starterPath)
stdoutMonitor = StdoutMonitor(process)
stderrMonitor = StderrMonitor(process)
stdoutMonitor.start()
stderrMonitor.start()
while self.isAlive(process):
Thread.sleep(300)
return self.getOutput(outfilePath)
finally:
os.remove(starterPath)
os.remove(outfilePath)
def logical_processor_count():
"""Returns the number of logical processors in the system.
"""
import multiprocessing
# The multiprocessing module provides support for Windows,
# BSD systems (including MacOS X) and systems which support
# the POSIX API for querying the number of CPUs.
try:
return multiprocessing.cpu_count()
except NotImplementedError:
pass
# For Jython, we need to query the Java runtime environment.
try:
from java.lang import Runtime
runtime = Runtime.getRuntime()
res = runtime.availableProcessors()
if res > 0:
return res
except ImportError:
pass
# Assuming that Solaris will support POSIX API for querying
# the number of CPUs. Just in case though, work it out by
# looking at the devices corresponding to the available CPUs.
try:
pseudoDevices = os.listdir('/devices/pseudo/')
expr = re.compile('^cpuid@[0-9]+$')
res = 0
for pd in pseudoDevices:
if expr.match(pd) is not None:
res += 1
if res > 0:
return res
except OSError:
pass
# Fallback to assuming only a single CPU.
return 1
def __init__(self, persist=None):
"""Start a gnuplot process.
Create a 'GnuplotProcess' object. This starts a gnuplot
program and prepares to write commands to it.
Keyword arguments:
'persist=1' -- start gnuplot with the '-persist' option,
(which leaves the plot window on the screen even after
the gnuplot program ends, and creates a new plot window
each time the terminal type is set to 'x11'). This
option is not available on older versions of gnuplot.
"""
if persist is None:
persist = GnuplotOpts.prefer_persist
command = [GnuplotOpts.gnuplot_command]
if persist:
if not test_persist():
raise ('-persist does not seem to be supported '
'by your version of gnuplot!')
command.append('-persist')
# This is a kludge: distutils wants to import everything it
# sees when making a distribution, and if we just call exec()
# normally that causes a SyntaxError in CPython because "exec"
# is a keyword. Therefore, we call the exec() method
# indirectly.
#self.process = Runtime.getRuntime().exec(command)
exec_method = getattr(Runtime.getRuntime(), 'exec')
self.process = exec_method(command)
self.outprocessor = OutputProcessor(
'gnuplot standard output processor',
self.process.getInputStream(), sys.stdout
)
self.outprocessor.start()
self.errprocessor = OutputProcessor(
'gnuplot standard error processor',
self.process.getErrorStream(), sys.stderr
)
self.errprocessor.start()
self.gnuplot = self.process.getOutputStream()
def logical_processor_count():
"""Returns the number of logical processors in the system.
"""
# The multiprocessing module provides support for Windows,
# BSD systems (including MacOS X) and systems which support
# the POSIX API for querying the number of CPUs.
try:
return multiprocessing.cpu_count()
except NotImplementedError:
pass
# For Jython, we need to query the Java runtime environment.
try:
from java.lang import Runtime
runtime = Runtime.getRuntime()
res = runtime.availableProcessors()
if res > 0:
return res
except ImportError:
pass
# Assuming that Solaris will support POSIX API for querying
# the number of CPUs. Just in case though, work it out by
# looking at the devices corresponding to the available CPUs.
try:
pseudoDevices = os.listdir('/devices/pseudo/')
expr = re.compile('^cpuid@[0-9]+$')
res = 0
for pd in pseudoDevices:
if expr.match(pd) != None:
res += 1
if res > 0:
return res
except OSError:
pass
# Fallback to assuming only a single CPU.
return 1
def getOsType():
NOT_WINDOWS = 'NOT WINDOWS'
try:
osNameTemp = os.getenv('OS',NOT_WINDOWS)
if not osNameTemp == NOT_WINDOWS and string.find(osNameTemp, 'Windows') == 0:
return OS_TYPE_WINDOWS
# non-Windows process
process = Runtime.getRuntime().exec("uname")
output = process.getInputStream() # process' output is our input
br = BufferedReader(InputStreamReader(output))
osName = br.readLine()
except Exception, error:
log.error("Could not determine operating system type: " + str(error))
def __init__(self, persist=None):
"""Start a gnuplot process.
Create a 'GnuplotProcess' object. This starts a gnuplot
program and prepares to write commands to it.
Keyword arguments:
'persist=1' -- start gnuplot with the '-persist' option,
(which leaves the plot window on the screen even after
the gnuplot program ends, and creates a new plot window
each time the terminal type is set to 'x11'). This
option is not available on older versions of gnuplot.
"""
if persist is None:
persist = GnuplotOpts.prefer_persist
command = [GnuplotOpts.gnuplot_command]
if persist:
if not test_persist():
raise Exception('-persist does not seem to be supported '
'by your version of gnuplot!')
command.append('-persist')
# This is a kludge: distutils wants to import everything it
# sees when making a distribution, and if we just call exec()
# normally that causes a SyntaxError in CPython because "exec"
# is a keyword. Therefore, we call the exec() method
# indirectly.
#self.process = Runtime.getRuntime().exec(command)
exec_method = getattr(Runtime.getRuntime(), 'exec')
self.process = exec_method(command)
self.outprocessor = OutputProcessor(
'gnuplot standard output processor',
self.process.getInputStream(), sys.stdout
)
self.outprocessor.start()
self.errprocessor = OutputProcessor(
'gnuplot standard error processor',
self.process.getErrorStream(), sys.stderr
)
self.errprocessor.start()
self.gnuplot = self.process.getOutputStream()
def check_count():
# Python 2.6+
try:
import multiprocessing
return multiprocessing.cpu_count()
except (ImportError, NotImplementedError):
pass
# https://github.com/giampaolo/psutil
try:
import psutil
return psutil.cpu_count() # psutil.NUM_CPUS on old versions
except (ImportError, AttributeError):
pass
# POSIX
try:
import os
res = int(os.sysconf('SC_NPROCESSORS_ONLN'))
if res > 0:
return str(res) + ' - POSIX'
except (AttributeError, ValueError):
pass
# Windows
try:
import os
res = int(os.environ['NUMBER_OF_PROCESSORS'])
if res > 0:
return str(res) + ' - Windows'
except (KeyError, ValueError):
pass
# jython
try:
from java.lang import Runtime
runtime = Runtime.getRuntime()
res = runtime.availableProcessors()
if res > 0:
return str(res) + ' - jython'
except ImportError:
pass
raise Exception('Can not determine number of CPUs on this system')
def load_eol_ids(inpath, tax):
System.gc()
rt = Runtime.getRuntime()
print '# Memory', rt.freeMemory()/(1024*1024), rt.totalMemory()/(1024*1024)
page_to_nodes = {}
node_to_pages = {}
with open(inpath, 'r') as infile:
print '| Processing EOL page ids file %s' % (inpath)
reader = csv.reader(infile)
row_count = 0
hits = 0
for row in reader:
row_count += 1
[page_id, idspace, source_id] = row
if row_count % 250000 == 0:
print row_count, row
qid = QualifiedId(idspace, source_id)
node = tax.lookupQid(qid)
if node != None: # and node.isPotentialOtu():
nodes = page_to_nodes.get(page_id)
if nodes == None:
nodes = [(node, qid)]
page_to_nodes[page_id] = nodes
elif not in_alist(node, nodes):
nodes.append((node, qid))
pages = node_to_pages.get(node)
if pages == None:
pages = [page_id]
node_to_pages[node] = pages
elif not page_id in pages:
pages.append(page_id)
print '| OTT nodes having at least one EOL page: %s' % len(node_to_pages)
print '| EOL page ids having at least one OTT node: %s' % len(page_to_nodes)
# Sort page ids for each OTT node (will use smallest one)
for node in node_to_pages:
pages = node_to_pages[node]
if len(pages) > 1:
pages.sort(key=int)
return (page_to_nodes, node_to_pages)
def getOsType():
NOT_WINDOWS = 'NOT WINDOWS'
try:
osNameTemp = os.getenv('OS', NOT_WINDOWS)
if not osNameTemp == NOT_WINDOWS and string.find(
osNameTemp, 'Windows') == 0:
return OS_TYPE_WINDOWS
# non-Windows process
process = Runtime.getRuntime().exec("uname")
output = process.getInputStream() # process' output is our input
br = BufferedReader(InputStreamReader(output))
osName = br.readLine()
except Exception, error:
log.error("Could not determine operating system type: " + str(error))
def cachedBlast(fileName,
blastLocation,
database,
eValue,
query,
pipeline,
force=False):
"""
Performs a blast search using the blastp executable and database in blastLocation on
the query with the eValue. The result is an XML file saved to fileName. If fileName
already exists the search is skipped. If remote is true then the search is done remotely.
"""
if not os.path.isfile(fileName) or force:
output = open(fileName, "w")
command = [
blastLocation + "/bin/blastp", "-evalue",
str(eValue), "-outfmt", "5", "-query", query, "-num_threads",
str(Runtime.getRuntime().availableProcessors()), "-db", database
]
blastProcess = subprocess.Popen(command, stdout=output)
while blastProcess.poll() == None:
if pipeline.exception:
print "Stopping in blast"
psProcess = subprocess.Popen(["ps", "aux"],
stdout=subprocess.PIPE)
awkProcess = subprocess.Popen(
["awk", "/" + " ".join(command).replace("/", "\\/") + "/"],
stdin=psProcess.stdout,
stdout=subprocess.PIPE)
for line in awkProcess.stdout:
subprocess.Popen(["kill", "-9", re.split(r"\s+", line)[1]])
output.close()
raise pipeline.exception
if blastProcess.poll() != 0:
raise OSError()
output.close()
try:
return parseBlast(fileName)
except SAXParseException:
print 'Retry'
return cachedBlast(fileName, blastLocation, database, eValue, query,
pipeline, True)
def startThreads(function, fractionCores = 1, wait = 0, arguments = None, nThreads = None):
threads = []
if nThreads == None:
threadRange = range(max(int(Runtime.getRuntime().availableProcessors() * fractionCores), 1))
else:
threadRange = range(nThreads)
IJ.log('ThreadRange = ' + str(threadRange))
for p in threadRange:
if arguments == None:
thread = threading.Thread(target = function)
else:
IJ.log('These are the arguments ' + str(arguments) + 'III type ' + str(type(arguments)))
thread = threading.Thread(group = None, target = function, args = arguments)
threads.append(thread)
thread.start()
IJ.log('Thread ' + str(p) + ' started')
time.sleep(wait)
for idThread, thread in enumerate(threads):
thread.join()
IJ.log('Thread ' + str(idThread) + 'joined')
def register(view_index, filepaths, modelclass, csv_dir, params):
n_threads = Runtime.getRuntime().availableProcessors()
exe = Executors.newFixedThreadPool(n_threads)
try:
name = "matrices-view-%i" % view_index
matrices = loadMatrices(name, csvdir)
if not matrices:
matrices = computeForwardTransforms(filepaths[view_index],
klb_loader, getCalibration,
csv_dir, exe, modelclass,
params)
saveMatrices(name, csv_dir)
finally:
exe.shutdown()
transforms = asBackwardConcatTransforms(matrices,
transformclass=Translation3D)
path_transforms = dict(izip(filepaths[view_index], transforms))
registered_loader = RegisteredLoader(klb_loader, path_transforms)
return Load.lazyStack(filepaths[view_index], registered_loader)
def exec(self,command):
""" Execute an OS command. This function also starts three threads
that handle the input, error and output streams. Then the other
functions can be used to make a conversation with the os process."""
rt = Runtime.getRuntime()
proc = rt.exec(command)
self.__outputwriter = DataOutputStream(proc.getOutputStream())
# start a new thread for the input stream of the process and set the
# Reader
self.__instr = StreamGobbler(proc.getInputStream())
self.__inputreader = Reader(BufferedReader(InputStreamReader(self.__instr)))
# start a new thread for error stream of the process and set the
# Reader
self.__errstr = StreamGobbler(proc.getErrorStream())
self.__errorreader = Reader(BufferedReader(InputStreamReader(self.__errstr)))
self.__process = proc
return proc
def run_CPython(path, script):
"""
Run CPython script from command prompt
Parameters
----------
path : str
Path to python script
script : str
Name of python script to run
Returns
-------
list
Output from stdout
list
Output from stderr
"""
# Command to run
cmd_str = "cmd /k cd {} & python {} & exit()".format(path, script)
# Start a new process, run python script from cmd
run = Runtime.getRuntime()
proc = run.exec(cmd_str)
# Collect output from stdout and stderr, print to console
stdout_reader = BufferedReader(InputStreamReader(proc.getInputStream()))
stderr_reader = BufferedReader(InputStreamReader(proc.getErrorStream()))
print("stdout: \n")
stdout = print_output(stdout_reader)
print("stderr: \n")
stderr = print_output(stderr_reader)
return stdout, stderr
def execute(cmd): runtime = Runtime.getRuntime() p = runtime.exec(cmd) p.outputStream.close() result = "" reader = BufferedReader(InputStreamReader(p.inputStream)) errorReader = BufferedReader(InputStreamReader(p.errorStream)) while True: if p.errorStream.available() > 0: print errorReader.readLine() line=reader.readLine() if line == None: break result+=line + "\n" while True: line = errorReader.readLine() if line == None: break print line p.waitFor() if p.exitValue() != 0: print result raise RuntimeError, 'execution failure' return result
def cpu_count():
"""Returns the number of CPUs in the system.
"""
# The multiprocessing module support Windows, BSD systems MacOS X and systems which support the POSIX API
try:
return multiprocessing.cpu_count()
except NotImplementedError:
pass
# For Jython, we need to query the Java runtime environment.
try:
from java.lang import Runtime
runtime = Runtime.getRuntime()
res = runtime.availableProcessors()
if res > 0:
return res
except ImportError:
pass
# Solaris system
try:
pseudoDevices = os.listdir('/devices/pseudo/')
expr = re.compile('^cpuid@[0-9]+$')
res = 0
for pd in pseudoDevices:
if expr.match(pd) != None:
res += 1
if res > 0:
return res
except OSError:
pass
return 1
def cpu_count_hack():
if os.name.startswith('posix'):
ncpus = os.sysconf('SC_NPROCESSORS_ONLN')
if isinstance(ncpus, int) and ncpus > 0:
return ncpus
elif sys.platform.startswith('win'):
ncpus = int(os.environ['NUMBER_OF_PROCESSORS']);
if ncpus > 0:
return ncpus
elif sys.platform.startswith('java'):
from java.lang import Runtime
runtime = Runtime.getRuntime()
ncpus = runtime.availableProcessors()
if ncpus > 0:
return ncpus
if sys.platform.startswith('linux'):
ncpus = int(os.popen2('grep -c processor /proc/cpuinfo')[1].read().strip())
if ncpus > 0:
return ncpus
if sys.platform.startswith('darwin'):
ncpus = int(os.popen2("sysctl -n hw.ncpu")[1].read())
if ncpus > 0:
return ncpus
return 1 # Default
def get_number_of_core():
"""
Number of virtual or physical CPUs on this system, i.e.
user/real as output by time(1) when called with an optimally scaling
userspace-only program.
Source code get on stackoverflow posted by phihag.
"""
# Python 2.6+
try:
import multiprocessing
return multiprocessing.cpu_count()
except (ImportError,NotImplementedError):
pass
# POSIX
try:
res = int(os.sysconf('SC_NPROCESSORS_ONLN'))
if res > 0:
return res
except (AttributeError,ValueError):
pass
# Windows
try:
res = int(os.environ['NUMBER_OF_PROCESSORS'])
if res > 0:
return res
except (KeyError, ValueError):
pass
# jython
try:
from java.lang import Runtime
runtime = Runtime.getRuntime()
res = runtime.availableProcessors()
if res > 0:
return res
except ImportError:
pass
# BSD
try:
sysctl = subprocess.Popen(['sysctl', '-n', 'hw.ncpu'],
stdout=subprocess.PIPE)
scStdout = sysctl.communicate()[0]
res = int(scStdout)
if res > 0:
return res
except (OSError, ValueError):
pass
# Linux
try:
res = open('/proc/cpuinfo').read().count('processor\t:')
if res > 0:
return res
except IOError:
pass
# Solaris
try:
pseudoDevices = os.listdir('/devices/pseudo/')
expr = re.compile('^cpuid@[0-9]+$')
res = 0
for pd in pseudoDevices:
if expr.match(pd) != None:
res += 1
if res > 0:
return res
except OSError:
pass
# Other UNIXes (heuristic)
try:
try:
dmesg = open('/var/run/dmesg.boot').read()
except IOError:
dmesgProcess = subprocess.Popen(['dmesg'], stdout=subprocess.PIPE)
dmesg = dmesgProcess.communicate()[0]
res = 0
while '\ncpu' + str(res) + ':' in dmesg:
res += 1
if res > 0:
return res
except OSError:
pass
raise Exception('Can not determine number of CPUs on this system')
def openURL(self,event):
try:
Desktop.getDesktop().browse(net.URI(self.url))
except:
Runtime.getRuntime().exec("xdg-open "+self.url)
def determineNumberOfCPUs():
""" Number of virtual or physical CPUs on this system, i.e.
user/real as output by time(1) when called with an optimally scaling
userspace-only program"""
# Python 2.6+
try:
import multiprocessing
return multiprocessing.cpu_count()
except (ImportError, NotImplementedError):
pass
# POSIX
try:
res = int(os.sysconf('SC_NPROCESSORS_ONLN'))
if res > 0:
return res
except (AttributeError, ValueError):
pass
# Windows
try:
res = int(os.environ['NUMBER_OF_PROCESSORS'])
if res > 0:
return res
except (KeyError, ValueError):
pass
# jython
try:
from java.lang import Runtime
runtime = Runtime.getRuntime()
res = runtime.availableProcessors()
if res > 0:
return res
except ImportError:
pass
# BSD
try:
sysctl = subprocess.Popen(['sysctl', '-n', 'hw.ncpu'],
stdout=subprocess.PIPE)
scStdout = sysctl.communicate()[0]
res = int(scStdout)
if res > 0:
return res
except (OSError, ValueError):
pass
# Linux
try:
res = open('/proc/cpuinfo').read().count('processor\t:')
if res > 0:
return res
except IOError:
pass
# Solaris
try:
pseudoDevices = os.listdir('/devices/pseudo/')
expr = re.compile('^cpuid@[0-9]+$')
res = 0
for pd in pseudoDevices:
if expr.match(pd) != None:
res += 1
if res > 0:
return res
except OSError:
pass
# Other UNIXes (heuristic)
try:
try:
dmesg = open('/var/run/dmesg.boot').read()
except IOError:
dmesgProcess = subprocess.Popen(['dmesg'], stdout=subprocess.PIPE)
dmesg = dmesgProcess.communicate()[0]
res = 0
while '\ncpu' + str(res) + ':' in dmesg:
res += 1
if res > 0:
return res
except OSError:
pass
raise Exception('Can not determine number of CPUs on this system')
if platform == 'win32':
d = tempfile.gettempdir()
default_prefix = d[0].upper() + d[1:]
else:
default_prefix = '/usr/local/'
default_jobs = os.environ.get('JOBS', -1)
if default_jobs < 1:
try:
if 'SC_NPROCESSORS_ONLN' in os.sysconf_names:
default_jobs = os.sysconf('SC_NPROCESSORS_ONLN')
else:
default_jobs = int(Utils.cmd_output(['sysctl', '-n', 'hw.ncpu']))
except:
if os.name == 'java':
from java.lang import Runtime
default_jobs = Runtime.getRuntime().availableProcessors()
else:
default_jobs = int(os.environ.get('NUMBER_OF_PROCESSORS', 1))
default_destdir = os.environ.get('DESTDIR', '')
def get_usage(self):
cmds_str = []
module = Utils.g_module
if module:
tbl = module.__dict__
keys = list(tbl.keys())
keys.sort()
if 'build' in tbl:
if not module.build.__doc__:
module.build.__doc__ = 'builds the project'
def available_cpu_count():
""" Number of available virtual or physical CPUs on this system, i.e.
user/real as output by time(1) when called with an optimally scaling
userspace-only program"""
import re
import subprocess
# cpuset
# cpuset may restrict the number of *available* processors
try:
m = re.search(r'(?m)^Cpus_allowed:\s*(.*)$',
open('/proc/self/status').read())
if m:
res = bin(int(m.group(1).replace(',', ''), 16)).count('1')
if res > 0:
return res
except IOError:
pass
# Python 2.6+
try:
import multiprocessing
return multiprocessing.cpu_count()
except (ImportError, NotImplementedError):
pass
# http://code.google.com/p/psutil/
try:
import psutil
return psutil.cpu_count() # psutil.NUM_CPUS on old versions
except (ImportError, AttributeError):
pass
# POSIX
try:
res = int(os.sysconf('SC_NPROCESSORS_ONLN'))
if res > 0:
return res
except (AttributeError, ValueError):
pass
# Windows
try:
res = int(os.environ['NUMBER_OF_PROCESSORS'])
if res > 0:
return res
except (KeyError, ValueError):
pass
# jython
try:
from java.lang import Runtime
runtime = Runtime.getRuntime()
res = runtime.availableProcessors()
if res > 0:
return res
except ImportError:
pass
# BSD
try:
sysctl = subprocess.Popen(['sysctl', '-n', 'hw.ncpu'],
stdout=subprocess.PIPE)
scStdout = sysctl.communicate()[0]
res = int(scStdout)
if res > 0:
return res
except (OSError, ValueError):
pass
# Linux
try:
res = open('/proc/cpuinfo').read().count('processor\t:')
if res > 0:
return res
except IOError:
pass
# Solaris
try:
pseudoDevices = os.listdir('/devices/pseudo/')
res = 0
for pd in pseudoDevices:
if re.match(r'^cpuid@[0-9]+$', pd):
res += 1
if res > 0:
return res
except OSError:
pass
# Other UNIXes (heuristic)
try:
try:
dmesg = open('/var/run/dmesg.boot').read()
except IOError:
dmesgProcess = subprocess.Popen(['dmesg'], stdout=subprocess.PIPE)
dmesg = dmesgProcess.communicate()[0]
res = 0
while '\ncpu' + str(res) + ':' in dmesg:
res += 1
if res > 0:
return res
except OSError:
pass
raise Exception('Can not determine number of CPUs on this system')
if operSystem.lower().startswith("windows"):
logger.info("Running on Windows platform ...")
# windows accepts the command in this format
command = ("""java -jar "%s" --config "%s" --debug DEBUG""" %
(atlantisPath, userConfig))
else:
# linux accepts the command in this format
logger.info("Running on non-Windows platform ...")
command = ("java -jar %s --config %s --debug DEBUG" %
(atlantisPath, userConfig))
logger.info("Running command: \"%s\"" % command)
# jython's os module doesn't define fork(), kill(), etc, then must
# use Java here
try:
rt = Runtime.getRuntime()
p = rt.exec(command)
logger.info("Waiting for the Atlantis process ...")
# (2008-12-10) on Windows: the above message is written out fine,
# Atlantis shows its splash window but hangs during start up.
# Suggestion is that input/output is not handled properly and should
# be read ...
# standard output ; standard error output streams handling
Thread(StreamReader(p.getInputStream(), logger)).start() # stdout
Thread(StreamReader(p.getErrorStream(), logger)).start() # stderr
# p.getOutputStream() - returns Output to the stream - is piped into
# the standard input stream (reference to the standard input of the
# process)
p.waitFor()
# Import Java Runtime, but call it "oscmd" to avoid conflict with MRL Runtime
from java.lang import Runtime as oscmd
process = oscmd.getRuntime().exec("notepad.exe")