def __init__(self):
threading.Thread.__init__(self)
self.myOverlay = None
sys.setcheckinterval(25)
self.chanlist = ChannelList()
self.paused = False
self.fullUpdating = True
def inject_jump(self, where, dest):
"""
Monkeypatch bytecode at ``where`` to force it to jump to ``dest``.
Returns function which puts things back how they were.
"""
# We're about to do dangerous things to a functions code content.
# We can't make a lock to prevent the interpreter from using those
# bytes, so the best we can do is to set the check interval to be high
# and just pray that this keeps other threads at bay.
old_check_interval = sys.getcheckinterval()
sys.setcheckinterval(2**20)
pb = ctypes.pointer(self.ob_sval)
orig_bytes = [pb[where+i][0] for i in xrange(where)]
v = struct.pack("<BH", opcode.opmap["JUMP_ABSOLUTE"], dest)
# Overwrite code to cause it to jump to the target
for i in xrange(3):
pb[where+i][0] = ord(v[i])
def tidy_up():
"""
Put the bytecode back how it was. Good as new.
"""
sys.setcheckinterval(old_check_interval)
for i in xrange(3):
pb[where+i][0] = orig_bytes[i]
return tidy_up
def test_setcheckinterval(self):
import sys
raises(TypeError, sys.setcheckinterval)
orig = sys.getcheckinterval()
for n in 0, 100, 120, orig: # orig last to restore starting state
sys.setcheckinterval(n)
assert sys.getcheckinterval() == n
def evaluate(self,pop,force = 0):
#import tree
#print '1',tree.ref()
#only send the individuals out that need evaluation
if force:
_eval_list = pop.data
else:
_eval_list = filter(lambda x: not x.evaluated,pop)
#print '2',tree.ref()
eval_list = pop.clone()
#print '3',tree.ref()
eval_list.data = _eval_list
if len(eval_list):
Nserv = len(pop.server_list)
groups = divide_list(eval_list,Nserv)
#print '4',tree.ref()
sys.setcheckinterval(10)
finished = sync.event()
bar = sync.barrier(Nserv)
#print "EVAL LENGTH!!!", plen(pop.evaluator)
gr = groups[0]
print "GROUP LENGTH!!!", plen(groups[0]), len(gr),
#print "IND!!!", plen(gr[0]),plen(gr[0].root)
#print '4.5',tree.ref()
for i in range(len(groups)):
inputs = {'sub_pop':groups[i], 'evaluator':pop.evaluator, 'force':force}
returns = ('sub_pop',)
code = 'evaluator.evaluate(sub_pop,force)'
data_pack = (inputs,returns,code)
server = pop.server_list[i]
thread.start_new_thread(remote_thread_eval,(bar,finished,server,data_pack))
#print '7',tree.ref()
finished.wait()
sys.setcheckinterval(10)
def tidy_up():
"""
Put the bytecode back how it was. Good as new.
"""
sys.setcheckinterval(old_check_interval)
for i in xrange(3):
pb[where+i][0] = orig_bytes[i]
def test_string_slicing(self):
def f(ct, passCt,chars):
x = "asdfasdf" * (ct / 8)
res = 0
for _ in xrange(passCt):
for ix in xrange(len(x)):
res = res + len(x[ix:ix+chars])
return res
self.evaluateWithExecutor(f, 1000000, 1, 2)
self.evaluateWithExecutor(f, 10000, 1, 2)
def runTest(func, name):
PerformanceTestReporter.PerfTest(name)(func)()
runTest(lambda: self.evaluateWithExecutor(f, 1000000, 10, 2), "pyfora.string_slicing_10mm.2_char_large_string.pyfora")
runTest(lambda: self.evaluateWithExecutor(f, 1000000, 10, 200), "pyfora.string_slicing_10mm.200_char_large_string.pyfora")
runTest(lambda: self.evaluateWithExecutor(f, 10000, 1000, 2), "pyfora.string_slicing_10mm.2_char_small_string.pyfora")
runTest(lambda: self.evaluateWithExecutor(f, 10000, 1000, 200), "pyfora.string_slicing_10mm.200_char_small_string.pyfora")
sys.setcheckinterval(100000)
runTest(lambda: f(1000000, 10, 2), "pyfora.string_slicing_10mm.2_char_large_string.native")
runTest(lambda: f(1000000, 10, 200), "pyfora.string_slicing_10mm.200_char_large_string.native")
runTest(lambda: f(10000, 1000, 2), "pyfora.string_slicing_10mm.2_char_small_string.native")
runTest(lambda: f(10000, 1000, 200), "pyfora.string_slicing_10mm.200_char_small_string.native")
sys.setcheckinterval(100)
def preprocessPercentileRatios(self):
print "preprocessPercentileRatios start"
distributionsFile = self.getDatasetSlidingSizesFile()
if os.path.isfile(distributionsFile):
#Teh distributions file is processed
print "The distribution file exists"
return
self.initializePropertiesComputeStructures(False)
print "computing the ratios"
try:
zpa = zipfile.ZipFile(distributionsFile,"w",zipfile.ZIP_DEFLATED)
zpa.writestr("dummy.txt","dummy file")
zpa.close()
self.ziplock = threading.Lock()
# Create a pool with three worker threads
pool = ThreadPool.ThreadPool(5)
sys.setcheckinterval(1000)
for windowSize in self.slidingWindowSizes:
if self.useWindowThreading:
pool.queueTask(self.preprocessWindowSize, windowSize, None)
else:
self.preprocessWindowSize(windowSize)
# When all tasks are finished, allow the threads to terminate
if self.useWindowThreading:
print "Joining the threads"
pool.joinAll()
except:
os.unlink(distributionsFile)
raise
print "preprocessPercentileRatios end"
def main():
"""Main entry point for the application"""
# Prepare for mantid import
prepare_mantid_env()
# todo: parse command arguments
# general initialization
app = initialize()
# the default sys check interval leads to long lags
# when request scripts to be aborted
sys.setcheckinterval(SYSCHECK_INTERVAL)
main_window = None
try:
main_window = start_workbench(app)
except BaseException:
# We count this as a crash
import traceback
# This is type of thing we want to capture and have reports
# about. Prints to stderr as we can't really count on anything
# else
traceback.print_exc(file=ORIGINAL_STDERR)
if main_window is None:
# An exception occurred don't exit here
return
ORIGINAL_SYS_EXIT()
def xdump(path, show_scheme=True, show_data=True):
# print "query_res " + str(xquery_res)
xobj, scheme, ret_type = list_path(path)
if xobj is None:
return None
if ret_type == "DIR":
ret_fields = [['dir']]
for (son_dir_name, son_dir) in xobj.items():
ret_fields.append([add_cross_if_dir(son_dir_name, son_dir)])
return ret_fields
ret_fields = list()
if show_scheme:
ret_fields.append(list(scheme.keys()))
if ret_type == "LOGS":
ret_fields.extend(xobj)
return ret_fields
def_interval = sys.getcheckinterval()
# TODO: maybe copy before and no need to lock?
sys.setcheckinterval(1000000000)
try:
ret_fields.extend(decompose_fields(xobj,
show_scheme=False,
show_data=show_data))
except Exception as e:
raise e
finally:
sys.setcheckinterval(def_interval)
return ret_fields
def test_string_slicing_into_vector(self):
def testFunction(ct, passCt,chars):
x = "asdfasdf" * (ct / 8)
res = 0
for _ in xrange(passCt):
v = [x[ix*chars:ix*chars+chars] for ix in xrange(len(x) / chars)]
for e in v:
res = res + len(e)
return res
f = testFunction
self.evaluateWithExecutor(f, 1000000, 1, 2)
self.evaluateWithExecutor(f, 10000, 1, 2)
def runTest(func, name):
PerformanceTestReporter.PerfTest(name)(func)()
runTest(lambda: self.evaluateWithExecutor(f, 1000000, 10, 2), "pyfora.string_slicing_into_vector_10mm.2_char_large_string.pyfora")
runTest(lambda: self.evaluateWithExecutor(f, 1000000, 1000, 200), "pyfora.string_slicing_into_vector_10mm.200_char_large_string.pyfora")
runTest(lambda: self.evaluateWithExecutor(f, 10000, 1000, 2), "pyfora.string_slicing_into_vector_10mm.2_char_small_string.pyfora")
runTest(lambda: self.evaluateWithExecutor(f, 10000, 100000, 200), "pyfora.string_slicing_into_vector_10mm.200_char_small_string.pyfora")
sys.setcheckinterval(100000)
runTest(lambda: f(1000000, 10, 2), "pyfora.string_slicing_into_vector_10mm.2_char_large_string.native")
runTest(lambda: f(1000000, 1000, 200), "pyfora.string_slicing_into_vector_10mm.200_char_large_string.native")
runTest(lambda: f(10000, 1000, 2), "pyfora.string_slicing_into_vector_10mm.2_char_small_string.native")
runTest(lambda: f(10000, 100000, 200), "pyfora.string_slicing_into_vector_10mm.200_char_small_string.native")
sys.setcheckinterval(100)
def setup(options):
sys.setcheckinterval(options.check_interval)
zope.app.appsetup.product.setProductConfigurations(
options.product_config)
options.eventlog()
options.accesslog()
for logger in options.loggers:
logger()
features = ('zserver',)
# Provide the devmode, if activated
if options.devmode:
features += ('devmode',)
logging.warning("Developer mode is enabled: this is a security risk "
"and should NOT be enabled on production servers. Developer mode "
"can be turned off in etc/zope.conf")
zope.app.appsetup.config(options.site_definition, features=features)
db = zope.app.appsetup.appsetup.multi_database(options.databases)[0][0]
notify(zope.processlifetime.DatabaseOpened(db))
task_dispatcher = ThreadedTaskDispatcher()
task_dispatcher.setThreadCount(options.threads)
for server in options.servers:
server.create(task_dispatcher, db)
notify(zope.processlifetime.ProcessStarting())
return db
def __init__(self):
threading.Thread.__init__(self)
self.myOverlay = None
self.shouldExit = False
sys.setcheckinterval(25)
self.chanlist = ChannelList()
self.chanlist.sleepTime = 0.1
def short_checkinterval(request):
"""
Sets a small interval using sys.setcheckinterval to cause many context
switches.
"""
old_interval = sys.getcheckinterval()
sys.setcheckinterval(0)
request.addfinalizer(lambda: sys.setcheckinterval(old_interval))
def test_setcheckinterval(self):
if test.test_support.due_to_ironpython_bug("http://tkbgitvstfat01:8080/WorkItemTracking/WorkItem.aspx?artifactMoniker=148342"):
return
self.assertRaises(TypeError, sys.setcheckinterval)
orig = sys.getcheckinterval()
for n in 0, 100, 120, orig: # orig last to restore starting state
sys.setcheckinterval(n)
self.assertEquals(sys.getcheckinterval(), n)
def test_setcheckinterval(self):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
self.assertRaises(TypeError, sys.setcheckinterval)
orig = sys.getcheckinterval()
for n in 0, 100, 120, orig: # orig last to restore starting state
sys.setcheckinterval(n)
self.assertEqual(sys.getcheckinterval(), n)
def doStatProf(*args):
import bench.statprof
sys.setcheckinterval(0)
bench.statprof.start()
runLimited(*args)
bench.statprof.stop()
bench.statprof.display()
def _collect(self):
gc.collect()
check_interval = sys.getcheckinterval()
sys.setcheckinterval(sys.maxint)
try:
return {id(object) for object in gc.get_objects() if not isinstance(object, EXCLUDE_TYPES)}
finally:
sys.setcheckinterval(check_interval)
def run(self):
pthread_setname_np(self.ident, "Manhole ----")
client = self.client
client.settimeout(None)
pid, uid, gid = get_peercred(client)
euid = os.geteuid()
client_name = "PID:%s UID:%s GID:%s" % (pid, uid, gid)
if uid not in (0, euid):
raise SuspiciousClient(
"Can't accept client with %s. "
"It doesn't match the current EUID:%s or ROOT." % (
client_name, euid
))
cry("Accepted connection %s from %s" % (client, client_name))
pthread_setname_np(self.ident, "Manhole %s" % pid)
client.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 0)
client.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 0)
backup = []
try:
client_fd = client.fileno()
for mode, names in (
('w', (
'stderr',
'stdout',
'__stderr__',
'__stdout__'
)),
('r', (
'stdin',
'__stdin__'
))
):
for name in names:
backup.append((name, getattr(sys, name)))
setattr(sys, name, os.fdopen(client_fd, mode, 0))
run_repl()
cry("DONE.")
finally:
cry("Cleaning up.")
old_interval = sys.getcheckinterval()
sys.setcheckinterval(2147483647)
junk = [] # keep the old file objects alive for a bit
for name, fh in backup:
junk.append(getattr(sys, name))
setattr(sys, name, fh)
del backup
for fh in junk:
try:
fh.close()
except IOError:
pass
del fh
del junk
self.client = None
sys.setcheckinterval(old_interval)
def thread_func(lock, acquire_array, lock_array, index):
gevent.sleep(random.uniform(0, 0.1))
# We use the fact that the GIL prevents CPython from context switching between the acquire_array.append()
# and the lock.acquire() when we set the check interval to a great value
sys.setcheckinterval(10000000)
acquire_array.append(index)
with lock:
sys.setcheckinterval(1)
lock_array.append(index)
def test_trashcan_threads(self):
# Issue #13992: trashcan mechanism should be thread-safe
NESTING = 60
N_THREADS = 2
def sleeper_gen():
"""A generator that releases the GIL when closed or dealloc'ed."""
try:
yield
finally:
time.sleep(0.000001)
class C(list):
# Appending to a list is atomic, which avoids the use of a lock.
inits = []
dels = []
def __init__(self, alist):
self[:] = alist
C.inits.append(None)
def __del__(self):
# This __del__ is called by subtype_dealloc().
C.dels.append(None)
# `g` will release the GIL when garbage-collected. This
# helps assert subtype_dealloc's behaviour when threads
# switch in the middle of it.
g = sleeper_gen()
next(g)
# Now that __del__ is finished, subtype_dealloc will proceed
# to call list_dealloc, which also uses the trashcan mechanism.
def make_nested():
"""Create a sufficiently nested container object so that the
trashcan mechanism is invoked when deallocating it."""
x = C([])
for i in range(NESTING):
x = [C([x])]
del x
def run_thread():
"""Exercise make_nested() in a loop."""
while not exit:
make_nested()
old_checkinterval = sys.getcheckinterval()
sys.setcheckinterval(3)
try:
exit = []
threads = []
for i in range(N_THREADS):
t = threading.Thread(target=run_thread)
threads.append(t)
with start_threads(threads, lambda: exit.append(1)):
time.sleep(1.0)
finally:
sys.setcheckinterval(old_checkinterval)
gc.collect()
self.assertEqual(len(C.inits), len(C.dels))
def setUp(self):
# Set a very small check interval, this will make it more likely
# that the interpreter crashes when threading is done incorrectly.
if sys.version_info[:2] >= (3, 2):
self._int = sys.getswitchinterval()
sys.setswitchinterval(0.0000001)
else:
self._int = sys.getcheckinterval()
sys.setcheckinterval(1)
def main():
"""Main entry point for the application"""
# setup command line arguments
parser = argparse.ArgumentParser(description='Mantid Workbench')
parser.add_argument('script', nargs='?')
parser.add_argument('-x', '--execute', action='store_true',
help='execute the script file given as argument')
parser.add_argument('-q', '--quit', action='store_true',
help='execute the script file with \'-x\' given as argument and then exit')
# TODO -a or --about: show about dialog and exit
# TODO -d or --default-settings: start MantidPlot with the default settings
# DONE -h or --help: show command line options <- free with command line parser
# TODO -v or --version: print MantidPlot version and release date
# TODO -r or --revision: print MantidPlot version and release date
# TODO -s or --silent: start mantidplot without any setup dialogs
# DONE -x or --execute: execute the script file given as argument
# DONE -xq or --executeandquit: execute the script file given as argument and then exit MantidPlot
# this is not a valid short command line option
try:
# set up bash completion as a soft dependency
import argcomplete
argcomplete.autocomplete(parser)
except ImportError:
pass # silently skip this
# parse the command line options
options = parser.parse_args()
# TODO handle options that don't require starting the workbench e.g. --help --version
# fix/validate arguments
if options.script is not None:
# convert into absolute path
options.script = os.path.abspath(os.path.expanduser(options.script))
if not os.path.exists(options.script):
# TODO should be logged
print('script "{}" does not exist'.format(options.script))
options.script = None
app = initialize()
# the default sys check interval leads to long lags
# when request scripts to be aborted
sys.setcheckinterval(SYSCHECK_INTERVAL)
exit_value = 0
try:
exit_value = start_workbench(app, options)
except BaseException:
# We count this as a crash
import traceback
# This is type of thing we want to capture and have reports
# about. Prints to stderr as we can't really count on anything
# else
traceback.print_exc(file=ORIGINAL_STDERR)
exit_value = -1
finally:
ORIGINAL_SYS_EXIT(exit_value)
def __init__(self, worker, workerArgs = None, syncs = []):
self.numWorkers = int(raw_input("Number of workers or cores: "))
sys.setcheckinterval(CHECK_INTERVAL)
self.worker = worker
self.workerArgs = workerArgs
self.sleep = SLEEP*60
self.syncEverySeconds = SYNC*60*60
self.syncs = syncs
self.syncRequest = False
def main():
'''
'''
sys.setcheckinterval(1000000)
try:
os.nice(-20)
except OSError:
print 'Note, running as root sets this script to a higher priority'
show_progress = True
print 'Reading file...'
fsBB, music = wav.read('classical1.wav')
music = music[0:2*fsBB] #Truncate to 2 seconds
musicL = music.T[0] #Separate out the left/right channels
musicR = music.T[1]
musicRL = musicL + musicR #merge the two channels
int16_max = float(2**15 - 1)
musicRL = musicRL/int16_max #normalize to 1
fsIF = 800000.
fc = 100000
BPL = True #Bandpass limiter, see demodulator
predeemph = True #Preemphasis and deemphasis filtering
fm_mod, kf = modulate_fm(musicRL, fsBB, fsIF, debug = False,
preemph = predeemph, fc = fc, progress = show_progress)
T = len(fm_mod)/fsIF
t = np.linspace(0, T, T*fsIF)
plt.plot(t[0:10000], fm_mod[0:10000])
plt.show()
return
# fm_mod = AWGN_channel(fm_mod, 10, debug = False, progress = show_progress)
# fm_mod = distortion_channel(fm_mod, taps = 10, progress = show_progress)
# fm_mod = multipath_channel(fm_mod, [1e-7],
# [1], fsIF, debug = False)
# fm_mod = nonlinearphase_channel(fm_mod, taps = 12, debug = True)
fm_mod = AWGN_channel(fm_mod, 0, debug = False, progress = show_progress)
#The analog_to_digital function is only really useful for generating an
#output file, since the following arithmetic is still done with float64.
#To model the effects of quantization just add gaussian noise...
#fm_mod = analog_to_digital(fm_mod, bits = 4, rnge = [-1,1], file_out = None,
# progress = show_progress)
BB_IQ = iqdemodulate_fm(fm_mod, fsIF, debug = False, deemph = predeemph, fc = fc,
BPL = BPL, progress = show_progress)
BB_ZX = zxdemodulate_fm(fm_mod, fsIF, debug = False, deemph = predeemph, fc = fc,
BPL = BPL, progress = show_progress)
play(BB_IQ, fsIF, fsBB, norm = 'inf', debug = False,
msg = 'Playing IQ, inf norm')
play(BB_ZX, fsIF, fsBB, norm = 'inf', debug = False,
msg = 'Playing ZX, inf norm')
return
def __init__(self, path=None):
"""Sets up and starts the `AppServer`.
`path` is the working directory for the AppServer
(directory in which AppServer is contained, by default)
This method loads plugins, creates the Application object,
and starts the request handling loop.
"""
self._running = 0
self._startTime = time()
global globalAppServer
if globalAppServer:
raise ProcessRunning("More than one AppServer" " or __init__() invoked more than once.")
globalAppServer = self
# Set up the import manager:
self._imp = ImportManager()
ConfigurableForServerSidePath.__init__(self)
if path is None:
path = os.path.dirname(__file__) # os.getcwd()
self._serverSidePath = os.path.abspath(path)
self._webKitPath = os.path.abspath(os.path.dirname(__file__))
self._webwarePath = os.path.dirname(self._webKitPath)
self.recordPID()
self._verbose = self.setting("Verbose")
if self._verbose:
self._silentURIs = self.setting("SilentURIs")
if self._silentURIs:
import re
self._silentURIs = re.compile(self._silentURIs)
else:
self._silentURIs = None
self._plugIns = []
self._requestID = 0
self.checkForInstall()
self.config() # cache the config
self.printStartUpMessage()
if self.setting("CheckInterval") is not None:
sys.setcheckinterval(self.setting("CheckInterval"))
self._app = self.createApplication()
self.loadPlugIns()
# @@ 2003-03 ib: shouldn't this just be in a subclass's __init__?
if self.isPersistent():
self._closeEvent = Event()
self._closeThread = Thread(target=self.closeThread, name="CloseThread")
# self._closeThread.setDaemon(1)
self._closeThread.start()
self._running = 1
def config(configfile, opts=None, dump=False):
""" initialize the application """
# pylint: disable = R0912, W0621
from wtf import config as _config
if configfile is None:
config = _config.Config(
_os.path.normpath(_os.path.abspath(_os.getcwd()))
)
else:
config = _config.load(configfile)
if dump:
_config.dump(config)
_sys.exit(0)
if 'wtf' in config and 'pythonpath' in config.wtf:
_sys.path = list(config.wtf.pythonpath) + _sys.path
if opts is not None and opts.checkinterval:
checkinterval = opts.checkinterval
elif 'wtf' in config:
checkinterval = config.wtf('checkinterval', 0)
else:
checkinterval = 0
if checkinterval:
_sys.setcheckinterval(checkinterval)
if opts is not None and opts.max_descriptors:
from wtf import cmdline as _cmdline
max_descriptors = opts.max_descriptors
exc = _cmdline.CommandlineError
elif 'wtf' in config:
max_descriptors = max(-1, int(config.wtf('max_descriptors', 0)))
exc = _config.ConfigurationError
else:
max_descriptors = 0
if max_descriptors:
try:
import resource as _resource
except ImportError:
raise exc(
"Cannot set max descriptors: resource module not available"
)
else:
try:
name = _resource.RLIMIT_NOFILE
except AttributeError:
try:
name = _resource.RLIMIT_OFILE
except AttributeError:
raise exc(
"Cannot set max descriptors: no rlimit constant found"
)
_resource.setrlimit(name, (max_descriptors, max_descriptors))
return config
def __unicode__(self):
"""Return a serialized version of this tree/branch."""
ci = getcheckinterval()
setcheckinterval(0)
value = ''.join(self.render('utf8')).decode('utf8')
setcheckinterval(ci)
return value
def pbar(title="Progress dialog example",
text="An informative message",
max=100,
style=wx.PD_CAN_ABORT|wx.PD_AUTO_HIDE|wx.PD_SMOOTH|wx.PD_ELAPSED_TIME|wx.PD_ESTIMATED_TIME|wx.PD_REMAINING_TIME,
*arg,**kwd):
sys.setcheckinterval(0)
gettext.install("wxpbr")
app = pbr(title,text,max,style,*arg,**kwd)
app.start()
return app
def tidy_up():
"""
Put the bytecode back to how it was. Good as new.
"""
if sys.version_info[0] < 3:
sys.setcheckinterval(old_check_interval)
else:
sys.setswitchinterval(old_check_interval)
for i in range(3):
pb[where + i][0] = orig_bytes[i]
def init ( self ):
# reduce check frequency to speed up things
sys.setcheckinterval(1000)
# check for the manager's SQLite database
if not os.path.exists(self.settings.dbPath):
print "Management database not found. Creating it..."
# connect to SQLite database and enable dictionary access of rows
self.dbCon = sqlite3.connect(self.settings.dbPath)
self.dbCon.row_factory = sqlite3.Row
# check if the dicom series database table exists
qResult = None
with self.dbCon:
dbCur = self.dbCon.cursor()
dbCur.execute( "SELECT name FROM sqlite_master WHERE type='table' AND name='%s'" % self.settings.dbTblSeries )
qResult = dbCur.fetchone()
if qResult is None:
# series table doesn't exist - create table in database
print "Database table '%s' not found. Creating it..." % self.settings.dbTblSeries
dcmHeaderNames = [ DicomDictionary[dcmHeaderKey][4] for dcmHeaderKey in self.settings.tagsToRecord ]
qCreate = "CREATE TABLE " + self.settings.dbTblSeries + " ( id INTEGER PRIMARY KEY, NumberOfDicoms INTEGER DEFAULT 0 NOT NULL, " + ' TEXT, '.join(dcmHeaderNames) + " TEXT )"
with self.dbCon:
dbCur = self.dbCon.cursor()
dbCur.execute(qCreate)
# check if the series' notes database table exists
qResult = None
with self.dbCon:
dbCur = self.dbCon.cursor()
dbCur.execute( "SELECT name FROM sqlite_master WHERE type='table' AND name='%s'" % self.settings.dbTblSeriesNotes )
qResult = dbCur.fetchone()
if qResult is None:
# projects table doesn't exist - create table in database
print "Database table '%s' not found. Creating it..." % self.settings.dbTblSeriesNotes
qCreate = ("CREATE TABLE %s ( id INTEGER PRIMARY KEY, SeriesInstanceUID INTEGER, Note TEXT ); "
"CREATE INDEX SeriesInstanceUidIdx ON %s (SeriesInstanceUID); "
) % ( self.settings.dbTblSeriesNotes, self.settings.dbTblSeriesNotes )
with self.dbCon:
dbCur = self.dbCon.cursor()
dbCur.executescript(qCreate)
# check root directory's existence
if not os.path.exists(self.settings.rootDir):
print "Root directory not found. You must create this!"
return
# check for dicom directory
if not os.path.exists(self.settings.dicomDir):
print "DICOM storage directory not found. Creating it..."
os.makedirs(self.settings.dicomDir)
if len(args) == 1:
func = conv3x3
else:
func = eval('conv3x3' + args[1])
n = 1000
for i in range(10):
func(Image(n, n), Image(3, 3))
if len(args) > 1:
return 'conv3x3%s(%s(%dx%d))' % (args[1], Image.__name__, n, n)
else:
return Image.__name__
if __name__ == '__main__':
import time, sys
sys.setcheckinterval(2**30)
try:
import pypyjit
pypyjit.set_param(trace_limit=200000)
except ImportError:
pass
Image = eval(sys.argv[1])
n = 1000
# Warmup
conv3x3(Image(n, n), Image(3, 3))
conv3x3iter(Image(n, n), Image(3, 3))
conv3x3range(Image(n, n), Image(3, 3))
a = time.time()
for i in range(10):
def sys_func():
lists = sys.argv # 传递给Python脚本的命令行参数列表 => python p.py -> ['p.py'] / python
strs = sys.getdefaultencoding() # 默认字符集名称
num = sys.getfilesystemencoding() # 系统文件名字符集名称
dicts = sys.getrefcount(object) # 返回object的引用计数（比实际多1个）
lists = sys.path # 模块搜索路径
sys.path.append("./test") # 动态添加模块搜索路径
strs = sys.platform # 平台标识符（系统身份进行详细的检查，推荐使用） Linux: 'linux' / Windows:'win32' / Cygwin:'cygwin' / Mac OS X:'darwin'
strs = sys.version # python解释器版本
lists = sys.thread_info # 线程信息
num = sys.api_version # 解释器C API版本
types, value, back = sys.exc_info() # 捕获异常 详见 异常 文章
sys.excepthook(types, value, back) # 打印异常
types = sys.last_type
value = sys.last_value
back = sys.last_trackback
# sys.exit([arg]) // 引发SystemExit异常退出Python(可以try)，范围[0,127], None==0, "string"==1
sys.exit(0)
num = sys.getrecursionlimit() # 最大递归数（堆栈最大深度）
sys.setrecursionlimit(5000) # 修改最大递归数
fnum = sys.getswitchinterval() # 获取线程切换间隔
sys.setswitchinterval(0.005) # 设置线程切换间隔，单位秒
num = sys.getcheckinterval() # 解释器的检查间隔
sys.setcheckinterval(100) # 设置解释器检查间隔，执行（默认）100个虚拟指令执行一次检查, 值为<=0时,检查每个虚拟指令
# sys.stdin // 标准输入流
strs = sys.stdin.readline()[:-1]
# sys.stdout // 标准出入输出
sys.stdout.write(">>")
sys.stdout.flush()
# sys.stderr // 标注错误流
sys.stderr.write(">>")
# ---
lists = sys.builtin_module_names # 所有模块 (注:非导入模块)
path = sys.base_exec_prefix # Python安装路径
path = sys.base_prefix # 同base_exec_prefix
path = sys.exec_prefix # 同base_exec_prefix
path = sys.prefix # 同base_exec_prefix
path = sys.executable # Python解释器的绝对路径
strs = sys.byteorder # 本机字节顺序指示器, big-endian(最高有效字节在第一位)值为'big', little-endian(最低有效字节在第一位)值为'little'
strs = sys.copyright # python版权
num = sys.hexversion # 16进制版本号
lists = sys.implementation # 当前运行的解释器的信息
num = sys.getallocatedblocks() # 解释器当前分配的内存块的数量
boolean = sys.dont_write_bytecode # 是否不会尝试导入源模块是写入.pyc文件 (False会写入.pyc文件)
# sys.getsizeof(object[, default]) // 返回对象的大小bit, 只计算自身内存消耗,不计算引用对象的内存消耗, 调用对象的__sizeof__(), default没有获取到默认返回值
num = sys.getsizeof(object)
boolean = sys.is_finalizing() # 解释器是否正在被关机
num = sys.maxsize # 最大整数值(2 ** 31 -1), 与系统有关
num = sys.maxunicode # 最大Unicode值的整数 (1114111)
strs = sys.ps1 # 解释器主提示符
strs = sys.ps2 # 解释器次提示符
sys.call_tracing(func, ("arg",)) # 调用函数
sys._clear_type_cache() # 清除内部类型缓存
sys._debugmallocstats() # 打印CPython内存分配器状态的低级信息
sys.setprofile(profilefunc) # 设置profile函数, 默认None
sys.getprofile() # 获取profile函数
sys.settrace(tracefunc) # 设置跟踪函数, def tracefunc(frame、event 和arg):
sys.gettrace() # 获取跟踪函数, 默认None
sys.set_coroutine_wrapper(wrapper) # 设置包装 def wrapper(coro):
sys.get_coroutine_wrapper() # 包装, 默认None
def tearDown(self):
sys.setcheckinterval(self._original_check_interval)
from OFS.Application import install_product
from OFS.Folder import Folder # NOQA
from Testing.ZopeTestCase import layer
# ZODB sandbox factory
from ZODB.DemoStorage import DemoStorage # NOQA
# Allow test authors to install Zope products into the test environment. Note
# that installProduct() must be called at module level -- never from tests.
from OFS.Application import get_folder_permissions # NOQA; NOQA
# Allow code to tell it is run by the test framework
os.environ['ZOPETESTCASE'] = '1'
# Increase performance on MP hardware
if PY2:
sys.setcheckinterval(2500)
# Always shut up
_quiet = True
def _print(msg):
'''Writes 'msg' to stderr and flushes the stream.'''
sys.stderr.write(msg)
sys.stderr.flush()
def _write(msg):
'''Writes 'msg' to stderr if not _quiet.'''
if not _quiet:
_print(msg)
def fastresponses(wakeup):
sys.setcheckinterval(64)
time.sleep(12.01)
if self.lastwakeup == wakeup:
sys.setcheckinterval(4096)
self.synsockets.clear()
import tkinter import model.core as core import gui.frame as frame import sys sys.setcheckinterval(0) core = core.Core() frame = frame.Frame(core)
# happened, then the path munging code is skipped.
swhome = r'INSERT_SOFTWARE_HOME'
import sys
import os
if swhome != 'INSERT_SOFTWARE_HOME' and EMMIX_DEBUG:
import sys
sys.path.insert(0, '%s/lib/python' % swhome)
sys.path.insert(1, '%s/bin/lib' % swhome)
sys.path.insert(2, '%s/bin/lib/plat-win' % swhome)
sys.path.insert(3, '%s/bin/lib/win32' % swhome)
sys.path.insert(4, '%s/bin/lib/win32/lib' % swhome)
sys.path.insert(5, '%s' % swhome)
import os, sys, getopt, string
sys.setcheckinterval(120)
program = sys.argv[0]
here = os.path.join(os.getcwd(), os.path.split(program)[0])
Zpid = ''
########################################################################
# Configuration section
## General configuration options
##
# If you want run as a daemon, then uncomment the line below:
if sys.platform == 'win32': Zpid = ''
else: Zpid = 'zProcessManager.pid'
def main(self):
rounds = self.values['-n']
reportfile = self.values['-f']
show_bench = self.values['-s']
regression_only = self.values['-r']
compare_to = self.values['-c']
hidenoise = self.values['-d']
warp = int(self.values['-w'])
withgc = self.values['--with-gc']
limitnames = self.values['-t']
if limitnames:
if _debug:
print '* limiting test names to one with substring "%s"' % \
limitnames
limitnames = re.compile(limitnames, re.I)
else:
limitnames = None
verbose = self.verbose
withsyscheck = self.values['--with-syscheck']
calibration_runs = self.values['-C']
timer = self.values['--timer']
print '-' * LINE
print 'PYBENCH %s' % __version__
print '-' * LINE
print '* using %s %s' % (getattr(
platform, 'python_implementation',
lambda: 'Python')(), string.join(string.split(sys.version), ' '))
# Switch off garbage collection
if not withgc:
try:
import gc
except ImportError:
print '* Python version doesn\'t support garbage collection'
else:
try:
gc.disable()
except NotImplementedError:
print '* Python version doesn\'t support gc.disable'
else:
print '* disabled garbage collection'
# "Disable" sys check interval
if not withsyscheck:
# Too bad the check interval uses an int instead of a long...
value = 2147483647
try:
sys.setcheckinterval(value)
except (AttributeError, NotImplementedError):
print '* Python version doesn\'t support sys.setcheckinterval'
else:
print '* system check interval set to maximum: %s' % value
if timer == TIMER_SYSTIMES_PROCESSTIME:
import systimes
print '* using timer: systimes.processtime (%s)' % \
systimes.SYSTIMES_IMPLEMENTATION
else:
print '* using timer: %s' % timer
print
if compare_to:
try:
f = open(compare_to, 'rb')
bench = pickle.load(f)
bench.name = compare_to
f.close()
compare_to = bench
except IOError, reason:
print '* Error opening/reading file %s: %s' % (
repr(compare_to), reason)
compare_to = None
def setupInterpreter(self):
""" make changes to the python interpreter environment """
sys.setcheckinterval(self.cfg.python_check_interval)
__version__ = '6.2.0' __author__ = 'Wim Lavrijsen ([email protected])' ### system and interpreter setup ------------------------------------------------ import os, sys, types import collections ## there's no version_info in 1.5.2 if sys.version[0:3] < '2.2': raise ImportError( 'Python Version 2.2 or above is required.' ) ## 2.2 has 10 instructions as default, > 2.3 has 100 ... make same if sys.version[0:3] == '2.2': sys.setcheckinterval( 100 ) ## readline support, if available try: import rlcompleter, readline class RootNameCompleter( rlcompleter.Completer ): def file_matches( self, text ): matches = [] path, name = os.path.split( text ) try: for fn in os.listdir( path or os.curdir ): if fn[:len(name)] == name: full = os.path.join( path, fn ) matches.append( full )
lc.printResults()
def testConflictSymmetricNoSync(self):
lc = LoopConflict(ops=[vmHostOpToggle] * 2,
args=self.args,
limit=self.loopLimit,
sync=False)
lc.run()
lc.printResults()
def testConflictAsymmetric(self):
lc = LoopConflict(ops=[vmHostOpOn, vmDelDisk],
args=self.args,
limit=self.loopLimit,
sync=False)
lc.run()
lc.printResults()
if __name__ == "__main__":
# set the check interval to a low number to increase concurrency for I/O bound ops
sys.setcheckinterval(1)
hostOpToggleSuite = unittest.TestSuite()
hostOpToggleSuite.addTest(HostOpToggleTestCase("testIndependent"))
# hostOpToggleSuite.addTest(HostOpToggleTestCase("testConflictSymmetric"))
# hostOpToggleSuite.addTest(HostOpToggleTestCase("testConflictSymmetricNoSync"))
# hostOpToggleSuite.addTest(HostOpToggleTestCase("testConflictAsymmetric"))
""" or, simply say - """
# hostOpToggleSuite = unittest.MakeSuite(HostOpToggleTestCase)
unittest.TextTestRunner(verbosity=2).run(hostOpToggleSuite)
def main():
metadata = {'pybench_version': __version__}
runner = perf.Runner(metadata=metadata, add_cmdline_args=add_cmdline_args)
cmd = runner.argparser
cmd.add_argument("-b",
"--benchmarks",
metavar="REGEX",
help='run only tests with names matching REGEX')
cmd.add_argument('--with-gc',
action="store_true",
help='enable garbage collection')
cmd.add_argument('--with-syscheck',
action="store_true",
help='use default sys check interval')
cmd.add_argument('--copyright', action="store_true", help='show copyright')
cmd.add_argument('--list',
action="store_true",
help='display the list of benchmarks and exit')
args = runner.parse_args()
bench = Benchmark(runner, args.output, verbose=runner.args.verbose)
bench.rounds = runner.args.warmups + runner.args.samples
if args.copyright:
print(__copyright__.strip())
print()
sys.exit()
if args.list:
bench.list_benchmarks(args)
sys.exit()
# Switch off garbage collection
if not args.with_gc:
try:
import gc
except ImportError:
print('* Python version doesn\'t support garbage collection',
file=sys.stderr)
else:
try:
gc.disable()
except NotImplementedError:
print('* Python version doesn\'t support gc.disable',
file=sys.stderr)
else:
if args.verbose:
print('* disabled garbage collection', file=sys.stderr)
# "Disable" sys check interval
if not args.with_syscheck:
# Too bad the check interval uses an int instead of a long...
value = 2147483647
try:
sys.setcheckinterval(value)
except (AttributeError, NotImplementedError):
print('* Python version doesn\'t support sys.setcheckinterval',
file=sys.stderr)
else:
if args.verbose:
print('* system check interval set to maximum: %s' % value,
file=sys.stderr)
bench.load_tests(runner.args, Setup)
try:
bench.run()
except KeyboardInterrupt:
print()
print('*** KeyboardInterrupt -- Aborting')
print()
return
def setUp(self):
self._original_check_interval = sys.getcheckinterval()
sys.setcheckinterval(1)
def test_setcheckinterval(self):
self.assertRaises(TypeError, sys.setcheckinterval)
orig = sys.getcheckinterval()
for n in 0, 100, 120, orig: # orig last to restore starting state
sys.setcheckinterval(n)
self.assertEquals(sys.getcheckinterval(), n)
def setupInterpreter(self):
# make changes to the python interpreter environment
if PY2:
# Check interval is gone in supported Python 3 versions.
sys.setcheckinterval(self.cfg.python_check_interval)
def main():
"""Main entry point for the application"""
# setup command line arguments
parser = argparse.ArgumentParser(description='Mantid Workbench')
parser.add_argument('script', nargs='?')
parser.add_argument('-x',
'--execute',
action='store_true',
help='execute the script file given as argument')
parser.add_argument(
'-q',
'--quit',
action='store_true',
help=
'execute the script file with \'-x\' given as argument and then exit')
# TODO -a or --about: show about dialog and exit
# TODO -d or --default-settings: start MantidPlot with the default settings
# DONE -h or --help: show command line options <- free with command line parser
# TODO -v or --version: print MantidPlot version and release date
# TODO -r or --revision: print MantidPlot version and release date
# TODO -s or --silent: start mantidplot without any setup dialogs
# DONE -x or --execute: execute the script file given as argument
# DONE -xq or --executeandquit: execute the script file given as argument and then exit MantidPlot
# this is not a valid short command line option
try:
# set up bash completion as a soft dependency
import argcomplete
argcomplete.autocomplete(parser)
except ImportError:
pass # silently skip this
# parse the command line options
options = parser.parse_args()
# TODO handle options that don't require starting the workbench e.g. --help --version
# fix/validate arguments
if options.script is not None:
# convert into absolute path
options.script = os.path.abspath(os.path.expanduser(options.script))
if not os.path.exists(options.script):
# TODO should be logged
print('script "{}" does not exist'.format(options.script))
options.script = None
app = initialize()
# the default sys check interval leads to long lags
# when request scripts to be aborted
sys.setcheckinterval(SYSCHECK_INTERVAL)
exit_value = 0
try:
exit_value = start_workbench(app, options)
except BaseException:
# We count this as a crash
import traceback
# This is type of thing we want to capture and have reports
# about. Prints to stderr as we can't really count on anything
# else
traceback.print_exc(file=ORIGINAL_STDERR)
exit_value = -1
finally:
ORIGINAL_SYS_EXIT(exit_value)
runner = perf.Runner(add_cmdline_args=add_cmdline_args)
runner.metadata[
'description'] = "Test the performance of Python's threads."
benchmarks = {
"iterative_count": test_iterative_count,
"threaded_count": test_threaded_count
}
parser = runner.argparser
parser.add_argument("--num_threads",
action="store",
type=int,
default=2,
dest="num_threads",
help="Number of threads to test.")
parser.add_argument("--check_interval",
action="store",
type=int,
default=sys.getcheckinterval(),
dest="check_interval",
help="Value to pass to sys.setcheckinterval().")
runner.argparser.add_argument("benchmark", choices=sorted(benchmarks))
options = runner.parse_args()
name = 'threading_%s' % options.benchmark
bench_func = benchmarks[options.benchmark]
sys.setcheckinterval(options.check_interval)
runner.bench_sample_func(name, bench_func, options.num_threads)
def main():
usage = "usage: %prog [-h|--help] [options]"
parser = OptionParser(usage=usage)
parser.add_option("-t",
"--throughput",
action="store_true",
dest="throughput",
default=False,
help="run throughput tests")
parser.add_option("-l",
"--latency",
action="store_true",
dest="latency",
default=False,
help="run latency tests")
parser.add_option("-b",
"--bandwidth",
action="store_true",
dest="bandwidth",
default=False,
help="run I/O bandwidth tests")
parser.add_option("-i",
"--interval",
action="store",
type="int",
dest="check_interval",
default=None,
help="sys.setcheckinterval() value")
parser.add_option("-I",
"--switch-interval",
action="store",
type="float",
dest="switch_interval",
default=None,
help="sys.setswitchinterval() value")
parser.add_option("-n",
"--num-threads",
action="store",
type="int",
dest="nthreads",
default=4,
help="max number of threads in tests")
# Hidden option to run the pinging and bandwidth clients
parser.add_option("",
"--latclient",
action="store",
dest="latclient",
default=None,
help=SUPPRESS_HELP)
parser.add_option("",
"--bwclient",
action="store",
dest="bwclient",
default=None,
help=SUPPRESS_HELP)
options, args = parser.parse_args()
if args:
parser.error("unexpected arguments")
if options.latclient:
kwargs = eval(options.latclient)
latency_client(**kwargs)
return
if options.bwclient:
kwargs = eval(options.bwclient)
bandwidth_client(**kwargs)
return
if not options.throughput and not options.latency and not options.bandwidth:
options.throughput = options.latency = options.bandwidth = True
if options.check_interval:
sys.setcheckinterval(options.check_interval)
if options.switch_interval:
sys.setswitchinterval(options.switch_interval)
print("== %s %s (%s) ==" % (
platform.python_implementation(),
platform.python_version(),
platform.python_build()[0],
))
# Processor identification often has repeated spaces
cpu = ' '.join(platform.processor().split())
print("== %s %s on '%s' ==" % (
platform.machine(),
platform.system(),
cpu,
))
print()
if options.throughput:
print("--- Throughput ---")
print()
run_throughput_tests(options.nthreads)
if options.latency:
print("--- Latency ---")
print()
run_latency_tests(options.nthreads)
if options.bandwidth:
print("--- I/O bandwidth ---")
print()
run_bandwidth_tests(options.nthreads)
def multi_processor(reader,
workflow_class,
graph_builder_constructor,
graph_builder_params,
ranker_constructor,
ranker_params,
eval_parameters,
ranking_cutoff,
mesh_tree_filename,
distance_matrix_filename,
distance_function,
umls_converter_data_filename,
umls_concept_data_filename,
extra_data_name,
extra_data_contents,
output_file,
num_processes=None,
queue_size=None,
output_callback=output,
output_headers_callback=output_headers,
output_item_callback=output_one_item,
performance_tuning=True):
"""
Perform the evaluation.
Multiprocessing notes: It's the responsibility of the caller to make sure that
extra_data_contents, if any, are multiprocessing-safe. For example, by using
a SyncManager and Namespace and passing the proxy. See umls/concept for an example.
"""
if num_processes is None:
num_processes = cpu_count()
if performance_tuning:
# Since reading the file involves an awful lot of object creation
# and destruction we'll tweak the gc adjustments to sweep less frequently
# IOW - we have a LOT of short-lived objects. No sense garbage-collecting
# the latter generations very often.
# (this is about 10x, 5x, and 5x the usual)
original_threshold = gc.get_threshold()
gc.set_threshold(10 * original_threshold[0], 5 * original_threshold[1],
5 * original_threshold[1])
original_check_interval = sys.getcheckinterval()
# Similarly, we'll try to minimize overhead from thread switches
# 5x usual value
sys.setcheckinterval(5 * original_check_interval)
logging.debug("Initializing Concept storage from %s",
umls_concept_data_filename)
if umls_concept_data_filename is None:
Concept.init_storage()
else:
Concept.init_storage(StringDBDict(umls_concept_data_filename))
Pmid.init_storage()
proctitle.setproctitle("MEDRank-main")
processes = []
logging.info("Creating %d worker processes.", num_processes)
#task_queue=[JoinableQueue(queue_size) for x in xrange(num_processes)]
task_queues = [Queue(queue_size) for x in xrange(num_processes)]
this_output_queue = Queue(2 * queue_size)
# Create an output processor
output_processor = Process(target=output_callback,
args=(output_file, this_output_queue,
output_headers_callback,
output_item_callback))
output_processor.start()
for i in xrange(num_processes):
this_process = Process(
target=processor,
args=(workflow_class, graph_builder_constructor,
graph_builder_params, ranker_constructor, ranker_params,
eval_parameters, ranking_cutoff, mesh_tree_filename,
distance_matrix_filename, distance_function,
umls_converter_data_filename, extra_data_name,
extra_data_contents, task_queues[i], this_output_queue,
"MEDRank-Worker-%d" % i),
name="MEDRank-Worker-%d" % i)
logging.log(ULTRADEBUG, "Created process: %r", this_process)
this_process.start()
processes.append((this_process, this_output_queue, task_queues[i]))
all_results = {}
count = 0
# Use a single dispatch queue for automagical load balancing
# CHANGED - Now uses multiple queues to avoid starving due to waiting on semlocks
for each_article in reader:
count += 1
#queues_and_sizes=[(task_queues[x].qsize(), x)
# for x in xrange(num_processes)]
#queues_and_sizes.sort()
#target_process=queues_and_sizes[0][1]
# logging.info("Dispatching article %d: %r", count, each_article)
target_process = (count - 1) % num_processes
#Lowest-loaded process first.
logging.info("Dispatching article %d: %s to %s", count,
each_article.set_id, processes[target_process][0].name)
task_queues[target_process].put(each_article)
#task_queue[target_process].put(each_article)
#task_queue.put(each_article)
#logging.info("The task queue is approximately %d items long.",
# task_queue.qsize())
logging.log(ULTRADEBUG, "Waiting for processing to end.")
all_results = {}
alive_processes = [x for x in processes if x[0].is_alive()]
remaining_processes = len(alive_processes)
logging.info("There are %d processes (out of %d) still alive.",
remaining_processes, num_processes)
for i in xrange(remaining_processes):
alive_processes[i][2].put('STOP')
alive_processes[i][2].close()
logging.debug("Sent STOP requests. Notifying queue that no further "
"requests will come.")
logging.info("All information sent to the processors.")
# Back to normal
if performance_tuning:
gc.set_threshold(original_threshold[0], original_threshold[1],
original_threshold[2])
sys.setcheckinterval(original_check_interval)
# Note end of output
while len(processes) > 0:
a_process = processes.pop()
# We join the process to wait for the end of the reading
a_process[0].join()
# logging.log(ULTRADEBUG, "Fetching results from finished process.")
# all_results.update(a_process[1].get()) # Add results to result pool
# logging.log(ULTRADEBUG, "Received results.")
logging.info("Finishing writing out results.")
this_output_queue.put("STOP")
output_processor.join()
logging.info("Results written. Finishing multiprocessing.")
return
def main():
sys.setcheckinterval(10000) # this is not threaded application
parseCmd.init()
# setup.py is FLASH_HOME/bin/setup.py
# FLASH_HOME is the parent of directory having setup.py
flashHome = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))
# Initialize other related directory names
GVars.init(flashHome)
# parse the command line arguments. The effect of calling
# this method is to set a lot of values in 'GVars'
parseCmd.parse()
############## Clean up simulation name ############################
os.chdir(GVars.simulationsDir)
if not os.path.isdir(GVars.simulationName): # Needs to be cleaned up?
# Find directories matching simulatation name (case insensitive)
simnames = dirGlob(GVars.simulationName + "*")
if len(simnames) == 1: # only one possible name
GVars.simulationName = simnames[0]
else:
if not simnames: # no match
raise SetupError("Unit %s not found" % GVars.simulationName)
else:
GVars.out.put(
"Potential completions : %s" % ', '.join(simnames),
globals.ERROR)
raise SetupError("Unit %s is ambiguous" % GVars.simulationName)
############ Proceed with other stuff ###################
os.chdir(GVars.flashHomeDir)
machDir = determineMachine()
unitList = UnitList(
) # A class which encapsulates operations of unit collections
objdir = os.path.join(GVars.flashHomeDir, GVars.objectDir)
if not os.path.isdir(objdir): os.mkdir(objdir)
if GVars.unitsFile: # copy over the user specified units file
sname = os.path.join(GVars.flashHomeDir, GVars.unitsFile)
tname = os.path.join(objdir, globals.UnitsFilename)
shutil.copy(sname, tname)
############## SWITCHING TO SOURCE DIR ############################
os.chdir(GVars.sourceDir)
if GVars.auto:
# generate and write an automatic "Units" file
unitList.generateUnitsfile()
# read units file, satisfy requirement and all the good stuf
unitList.populate()
# adjust values of command line options based on units used
# this is the only place the setup script uses its knowledge
# about Flash Units and what they do
unitList.adjustOpts()
parseCmd.final() # finalise the options
# create the object which does stuff relating to linking files
linkList = LinkFileList(objdir)
# Combine info from all Config files
configInfo = unitList.getConfigInfo()
# get Runtime Paramas info
rpInfo = unitList.getRPInfo(configInfo['max_plot_vars'])
# get info regarding all variables
varInfo = unitList.getVarInfo()
# Add library options to configInfo
for key in GVars.withLibraries.keys():
if not configInfo['LIBRARY'].has_key(key):
configInfo['LIBRARY'][key] = []
configInfo['LIBRARY'][key].append(GVars.withLibraries[key])
######### Now handle libraries depending on other libraries ##########
configInfo['libConfigInfo'] = LibUnion(configInfo['LIBRARY'])
# ConfigInfo['libConfigInfo'] is a dictionary mapping libraries to their config information
# also CLASS.libOrder is a list of pairs (libname, args)
# This order is important due to linker invocation order
configInfo['libConfigInfo'].writeLibraries(
) # write out lib data to setup_libraries
# only now is noClobber flag stable
linkList.cleanObjectDir()
############################### CHANGE TO OBJECT DIR ###################
os.chdir(GVars.flashHomeDir)
os.chdir(GVars.objectDir)
# remove the SuccessFile
try:
os.unlink(globals.SuccessFilename)
except OSError:
pass
rpInfo.writeCode(configInfo) # write Fortran code for RP support
# find files which should not be linked
linkList.getDontLinkList(unitList.getList(), configInfo['LINKIF'])
# getLinkOrder does the fancy sorting of unitnames
for unitname in unitList.getLinkOrder():
linkList.linkFiles(os.path.join(GVars.sourceDir, unitname))
linkList.linkFiles(os.path.join(GVars.flashHomeDir, 'sites'))
# Link in the right version of the make file
# attempt to find Makefile in FLASH root directory, otherwise link make file from sites
tmp_makefilePathRoot = os.path.join(flashHome,
"Makefile.h" + GVars.makefileext)
tmp_makefilePathSites = os.path.join(machDir,
"Makefile.h" + GVars.makefileext)
if os.path.isfile(tmp_makefilePathRoot):
linkList.addLink(tmp_makefilePathRoot, "Makefile.h")
print "Using Makefile.h: " + tmp_makefilePathRoot
else:
linkList.addLink(tmp_makefilePathSites, "Makefile.h")
print "Using Makefile.h: " + tmp_makefilePathSites
# functions in the simulations dir override earlier instances
linkList.linkFiles(GVars.simulationsDir)
# functions from LINKIF statements override everything else,
# assuming their conditions apply.
linkList.doLINKIFOverrides(unitList.getList(), configInfo['LINKIF'])
# now is when we do the real link/copying
linkList.reallyLink()
############## flash.par and Makefiles **************
# Copy in flash.par
if os.path.isfile(
os.path.join(GVars.simulationsDir, GVars.simulationName,
'flash.par')):
shutil.copy(
os.path.join(GVars.simulationsDir, GVars.simulationName,
'flash.par'), '.')
test_src = os.path.join(GVars.flashHomeDir, 'tools', 'scripts', 'testing',
'precision_test', 'precision_test.F90')
if os.path.isfile(test_src):
shutil.copy(test_src, '.')
GVars.out.put('creating Makefiles for all units', globals.INFO)
unitList.createMakefiles()
#FIXME merge with createMakefiles
GVars.out.put('generating buildstamp generator', globals.INFO)
generateBuildstampGenerator()
GVars.out.put('copying release accessor function Makefile', globals.INFO)
shutil.copy(os.path.join(GVars.flashHomeDir, 'bin/make_release'), '.')
GVars.out.put('copying buildstats accessor function Makefile',
globals.INFO)
shutil.copy(os.path.join(GVars.flashHomeDir, 'bin/make_bstats'), '.')
GVars.out.put('copying flashUnits accessor function Makefile',
globals.INFO)
unitList.generateSetupFlashUnits()
#shutil.copy(os.path.join(GVars.flashHomeDir, 'bin/make_flashUnits'), '.')
shutil.copy(os.path.join(GVars.flashHomeDir, 'bin/resetup'), '.')
GVars.out.put('copying Dependency generator', globals.INFO)
flist = [
"setup_depends.py", "setup_addcdepends.py", "setup_reorder.py",
"reorder.tpl"
]
for f in flist:
shutil.copy(os.path.join(GVars.flashHomeDir, 'bin/' + f), '.')
GVars.out.put('generating Makefile', globals.IMPINFO)
generateMakefile(configInfo, machDir)
# generateDbaseDefines(configInfo, opts)
generateFlashDefines(configInfo) # writes the Flash.h file
writeSimulationFiles(configInfo)
# Copy/link datafiles to object directory
if GVars.portable:
func = lambda real, link: shutil.copy2(real, link)
else:
cwd = os.getcwd()
func = lambda real, link: os.symlink(getRelPath(real, cwd), link)
datafiles = []
for wildcard in configInfo['DATAFILES']:
for file in glob.glob(os.path.join(GVars.sourceDir, wildcard)):
datafiles.append(file[len(GVars.sourceDir):].strip("/"))
bname = os.path.basename(file)
try:
func(file, bname)
except:
os.remove(bname)
func(file, bname)
simDir = os.path.join(GVars.simulationsDir, GVars.simulationName)
for wildcard in GVars.datafiles:
for file in glob.glob(os.path.join(simDir, wildcard)):
datafiles.append(file[len(GVars.sourceDir):].strip("/"))
bname = os.path.basename(file)
try:
func(file, bname)
except:
os.remove(bname)
func(file, bname)
if datafiles:
GVars.out.put('Copying data files: %d copied' % len(datafiles),
globals.IMPINFO)
# write "setup_datafiles" into object directory, even if empty
open(
os.path.join(GVars.flashHomeDir, GVars.objectDir,
globals.SetupDatafilesFilename),
'w').write("\n".join(datafiles))
if (GVars.setupVars.get("ParameshLibraryMode") or
((not GVars.setupVars.get("Grid") or
(GVars.setupVars.get("Grid").upper() == "PM4DEV")) and
(configInfo['PPDEFINES'].has_key('USE_AMR_RUNTIME_PARAMETERS_FILE') or
('-DUSE_AMR_RUNTIME_PARAMETERS_FILE' in GVars.definesNames)))):
generatePm3RPDefines(configInfo)
if GVars.parfile:
shutil.copy2(os.path.join(simDir, GVars.parfile), "flash.par")
GVars.out.put("Copied %s as flash.par" % GVars.parfile,
globals.IMPINFO)
parseCmd.writeCmdLine()
unitList.writeSetupUnitsFile()
varInfo.writeVarInfo()
# rp Stuff
rpInfo.writeRPInfo()
rpInfo.writeDefaultPar()
# if no flash.par copy default.par over
if not os.path.isfile('flash.par'):
shutil.copy(globals.RPDefaultParFilename, 'flash.par')
# create the successfile
ofd = open(globals.SuccessFilename, "w")
ofd.write("SUCCESS\n")
ofd.close()
os.chdir(GVars.flashHomeDir)
GVars.out.put('SUCCESS', globals.ALWAYS)
def done(self):
sys.setcheckinterval(4096)
import utils
import global_config
def _run_time_waster():
print('')
print('****** Starting time-waster ******')
while True:
try:
for i in range(0, 30000):
s = "a;djfalfjasdl;fajdl;fjasdf alkdfja ;ldfjal;sdfj jad;lfjasd;lfjaksdflkjd"
s = "qoroqerueouewqroiq nqeoiruq proqieu ruqoweruq eporuuqeropque r uuqoperuqowie"
s = "a;djfalfjasdl;fajdl;fjasdf alkdfja ;ldfjal;sdfj jad;lfjasd;lfjaksdflkjd"
s = "qoroqerueouewqroiq nqeoiruq proqieu ruqoweruq eporuuqeropque r uuqoperuqowie"
s = "a;djfalfjasdl;fajdl;fjasdf alkdfja ;ldfjal;sdfj jad;lfjasd;lfjaksdflkjd"
s = "qoroqerueouewqroiq nqeoiruq proqieu ruqoweruq eporuuqeropque r uuqoperuqowie"
except KeyboardInterrupt:
print('Got SIGINT (shutting down)')
break
except:
print('Got an unexpected exception')
sys.exit(1)
print('****** Exiting time-waster ******')
if __name__ == '__main__':
sys.setcheckinterval(global_config.check_interval)
_run_time_waster()
if len(failure) >= 2:
return None
print >> sys.stderr, 'Connecting to the meta-server %s:%d...' % METASERVER
try:
s = socket(AF_INET, SOCK_STREAM)
s.connect(METASERVER)
except error, e:
print >> sys.stderr, '*** cannot contact meta-server:', str(e)
failure.append(e)
return None
else:
print >> sys.stderr, 'connected.'
return s
sys.setcheckinterval(4096)
def float2str(f):
# don't trust locale issues and write a string with a '.'
s = str(long(f * 1000000.0))
return s[:-6] + '.' + s[-6:]
def str2float(s):
try:
return float(s)
except:
# locale issues may prevent float() from decoding the string
s = s.strip()
try:
def main(self):
rounds = self.values['-n']
reportfile = self.values['-f']
show_bench = self.values['-s']
compare_to = self.values['-c']
hidenoise = self.values['-d']
warp = int(self.values['-w'])
withgc = self.values['--with-gc']
limitnames = self.values['-t']
if limitnames:
if _debug:
print('* limiting test names to one with substring "%s"' % \
limitnames)
limitnames = re.compile(limitnames, re.I)
else:
limitnames = None
verbose = self.verbose
withsyscheck = self.values['--with-syscheck']
calibration_runs = self.values['-C']
timer = self.values['--timer']
print('-' * LINE)
print('PYBENCH %s' % __version__)
print('-' * LINE)
print('* using %s %s' % (
getattr(platform, 'python_implementation', lambda:'Python')(),
' '.join(sys.version.split())))
# Switch off garbage collection
if not withgc:
try:
import gc
except ImportError:
print('* Python version doesn\'t support garbage collection')
else:
try:
gc.disable()
except NotImplementedError:
print('* Python version doesn\'t support gc.disable')
else:
print('* disabled garbage collection')
# "Disable" sys check interval
if not withsyscheck:
# Too bad the check interval uses an int instead of a long...
value = 2147483647
try:
sys.setcheckinterval(value)
except (AttributeError, NotImplementedError):
print('* Python version doesn\'t support sys.setcheckinterval')
else:
print('* system check interval set to maximum: %s' % value)
if timer == TIMER_SYSTIMES_PROCESSTIME:
import systimes
print('* using timer: systimes.processtime (%s)' % \
systimes.SYSTIMES_IMPLEMENTATION)
else:
# Check that the clock function does exist
try:
get_timer(timer)
except TypeError:
print("* Error: Unknown timer: %s" % timer)
return
print('* using timer: %s' % timer)
if hasattr(time, 'get_clock_info'):
info = time.get_clock_info(timer[5:])
print('* timer: resolution=%s, implementation=%s'
% (info.resolution, info.implementation))
print()
if compare_to:
try:
f = open(compare_to,'rb')
bench = pickle.load(f)
bench.name = compare_to
f.close()
compare_to = bench
except IOError as reason:
print('* Error opening/reading file %s: %s' % (
repr(compare_to),
reason))
compare_to = None
if show_bench:
try:
f = open(show_bench,'rb')
bench = pickle.load(f)
bench.name = show_bench
f.close()
bench.print_header()
if compare_to:
bench.print_comparison(compare_to,
hidenoise=hidenoise,
limitnames=limitnames)
else:
bench.print_benchmark(hidenoise=hidenoise,
limitnames=limitnames)
except IOError as reason:
print('* Error opening/reading file %s: %s' % (
repr(show_bench),
reason))
print()
return
if reportfile:
print('Creating benchmark: %s (rounds=%i, warp=%i)' % \
(reportfile, rounds, warp))
print()
# Create benchmark object
bench = Benchmark(reportfile,
verbose=verbose,
timer=timer,
warp=warp,
calibration_runs=calibration_runs)
bench.rounds = rounds
bench.load_tests(Setup, limitnames=limitnames)
try:
bench.calibrate()
bench.run()
except KeyboardInterrupt:
print()
print('*** KeyboardInterrupt -- Aborting')
print()
return
bench.print_header()
if compare_to:
bench.print_comparison(compare_to,
hidenoise=hidenoise,
limitnames=limitnames)
else:
bench.print_benchmark(hidenoise=hidenoise,
limitnames=limitnames)
# Ring bell
sys.stderr.write('\007')
if reportfile:
try:
f = open(reportfile,'wb')
bench.name = reportfile
pickle.dump(bench,f)
f.close()
except IOError as reason:
print('* Error opening/writing reportfile')
except IOError as reason:
print('* Error opening/writing reportfile %s: %s' % (
reportfile,
reason))
print()
def tearDown(self):
if sys.version_info[:2] >= (3, 2):
sys.setswitchinterval(self._int)
else:
sys.setcheckinterval(self._int)
def setup_server(self):
"""Configure CherryPy server from application configuration.
Traverses the server configuration portion and applies parameters
known to be for CherryPy to the CherryPy server configuration.
These are: engine, hooks, log, request, respose, server, tools,
wsgi, checker.
Also applies pseudo-parameters ``thread_stack_size`` (default: 128kB)
and ``sys_check_interval`` (default: 10000). The former sets the
default stack size to desired value, to avoid excessively large
thread stacks -- typical operating system default is 8 MB, which
adds up rather a lot for lots of server threads. The latter sets
python's ``sys.setcheckinterval``; the default is to increase this
to avoid unnecessarily frequent checks for python's GIL, global
interpreter lock. In general we want each thread to complete as
quickly as possible without making unnecessary checks."""
cpconfig = cherrypy.config
# Determine server local base.
port = getattr(self.srvconfig, 'port', 8080)
local_base = getattr(self.srvconfig, 'local_base',
socket.gethostname())
if local_base.find(':') == -1:
local_base = '%s:%d' % (local_base, port)
# Set default server configuration.
cherrypy.log = Logger()
cpconfig.update({'server.max_request_body_size': 0})
cpconfig.update({'server.environment': 'production'})
cpconfig.update({'server.socket_host': '0.0.0.0'})
cpconfig.update({'server.socket_port': port})
cpconfig.update({'server.socket_queue_size': 100})
cpconfig.update({'server.thread_pool': 100})
cpconfig.update({'tools.proxy.on': True})
cpconfig.update({'tools.proxy.base': local_base})
cpconfig.update({'tools.time.on': True})
cpconfig.update({'engine.autoreload.on': False})
cpconfig.update({'request.show_tracebacks': False})
cpconfig.update(
{'request.methods_with_bodies': ("POST", "PUT", "DELETE")})
thread.stack_size(
getattr(self.srvconfig, 'thread_stack_size', 128 * 1024))
sys.setcheckinterval(
getattr(self.srvconfig, 'sys_check_interval', 10000))
self.silent = getattr(self.srvconfig, 'silent', False)
# Apply any override options from app config file.
for section in ('engine', 'hooks', 'log', 'request', 'response',
'server', 'tools', 'wsgi', 'checker'):
if not hasattr(self.srvconfig, section):
continue
for opt, value in getattr(self.srvconfig,
section).dictionary_().iteritems():
if isinstance(value, ConfigSection):
for xopt, xvalue in value.dictionary_().iteritems():
cpconfig.update(
{"%s.%s.%s" % (section, opt, xopt): xvalue})
elif isinstance(value, str) or isinstance(value, int):
cpconfig.update({"%s.%s" % (section, opt): value})
else:
raise RuntimeError(
"%s.%s should be string or int, got %s" %
(section, opt, type(value)))
# Apply security customisation.
if hasattr(self.srvconfig, 'authz_defaults'):
defsec = self.srvconfig.authz_defaults
cpconfig.update({'tools.cms_auth.on': True})
cpconfig.update({'tools.cms_auth.role': defsec['role']})
cpconfig.update({'tools.cms_auth.group': defsec['group']})
cpconfig.update({'tools.cms_auth.site': defsec['site']})
if hasattr(self.srvconfig, 'authz_policy'):
cpconfig.update(
{'tools.cms_auth.policy': self.srvconfig.authz_policy})
cherrypy.log("INFO: final CherryPy configuration: %s" %
pformat(cpconfig))
def main():
'''
'''
sys.setcheckinterval(1000000)
try:
os.nice(-20)
except OSError:
print 'Note, running as root sets this script to a higher priority'
show_progress = True
print 'Reading file...'
fsBB, music = wav.read('classical1.wav')
music = music[0:2 * fsBB] #Truncate to 2 seconds
musicL = music.T[0] #Separate out the left/right channels
musicR = music.T[1]
musicRL = musicL + musicR #merge the two channels
int16_max = float(2**15 - 1)
musicRL = musicRL / int16_max #normalize to 1
fsIF = 800000.
fc = 100000
BPL = True #Bandpass limiter, see demodulator
predeemph = True #Preemphasis and deemphasis filtering
fm_mod, kf = modulate_fm(musicRL,
fsBB,
fsIF,
debug=False,
preemph=predeemph,
fc=fc,
progress=show_progress)
T = len(fm_mod) / fsIF
t = np.linspace(0, T, T * fsIF)
plt.plot(t[0:10000], fm_mod[0:10000])
plt.show()
return
# fm_mod = AWGN_channel(fm_mod, 10, debug = False, progress = show_progress)
# fm_mod = distortion_channel(fm_mod, taps = 10, progress = show_progress)
# fm_mod = multipath_channel(fm_mod, [1e-7],
# [1], fsIF, debug = False)
# fm_mod = nonlinearphase_channel(fm_mod, taps = 12, debug = True)
fm_mod = AWGN_channel(fm_mod, 0, debug=False, progress=show_progress)
#The analog_to_digital function is only really useful for generating an
#output file, since the following arithmetic is still done with float64.
#To model the effects of quantization just add gaussian noise...
#fm_mod = analog_to_digital(fm_mod, bits = 4, rnge = [-1,1], file_out = None,
# progress = show_progress)
BB_IQ = iqdemodulate_fm(fm_mod,
fsIF,
debug=False,
deemph=predeemph,
fc=fc,
BPL=BPL,
progress=show_progress)
BB_ZX = zxdemodulate_fm(fm_mod,
fsIF,
debug=False,
deemph=predeemph,
fc=fc,
BPL=BPL,
progress=show_progress)
play(BB_IQ,
fsIF,
fsBB,
norm='inf',
debug=False,
msg='Playing IQ, inf norm')
play(BB_ZX,
fsIF,
fsBB,
norm='inf',
debug=False,
msg='Playing ZX, inf norm')
return
if sys.platform == 'win32':
# include modules directory (if it isn't already and it exists)
loc2 = os.environ['PYMOL_PATH'] + '/modules'
if os.path.exists(loc2):
if loc2 not in sys.path:
sys.path.insert(0, loc2)
# include installed numpy
loc1 = os.environ['PYMOL_PATH'] + '/modules/numeric'
if os.path.exists(loc1):
sys.path.insert(0, loc1)
if '' not in sys.path: # make sure cwd is in path like normal Python
sys.path.insert(0, '')
sys.setcheckinterval(1) # maximize responsiveness
# auto-detect bundled FREEMOL (if present)
if not os.environ.has_key("FREEMOL"):
test_path = os.path.join(os.environ['PYMOL_PATH'], "freemol")
if os.path.isdir(test_path):
os.environ['FREEMOL'] = test_path
# include FREEMOL's libpy in sys.path (if present)
if os.environ.has_key("FREEMOL"):
freemol_libpy = os.path.join(os.environ['FREEMOL'], "libpy")
if os.path.isdir(freemol_libpy):
if freemol_libpy not in sys.path:
sys.path.append(freemol_libpy)
def main(builtinParameters={}):
# Bump the GIL check interval, its more important to get any one thread to a
# blocking operation (hopefully exec) than to try and unblock other threads.
#
# FIXME: This is a hack.
import sys
sys.setcheckinterval(1000)
global options
from optparse import OptionParser, OptionGroup
parser = OptionParser("usage: %prog [options] {file-or-path}")
parser.add_option("-j",
"--threads",
dest="numThreads",
metavar="N",
help="Number of testing threads",
type=int,
action="store",
default=None)
parser.add_option("",
"--config-prefix",
dest="configPrefix",
metavar="NAME",
help="Prefix for 'lit' config files",
action="store",
default=None)
parser.add_option("",
"--param",
dest="userParameters",
metavar="NAME=VAL",
help="Add 'NAME' = 'VAL' to the user defined parameters",
type=str,
action="append",
default=[])
group = OptionGroup(parser, "Output Format")
# FIXME: I find these names very confusing, although I like the
# functionality.
group.add_option("-q",
"--quiet",
dest="quiet",
help="Suppress no error output",
action="store_true",
default=False)
group.add_option("-s",
"--succinct",
dest="succinct",
help="Reduce amount of output",
action="store_true",
default=False)
group.add_option("-v",
"--verbose",
dest="showOutput",
help="Show all test output",
action="store_true",
default=False)
group.add_option("",
"--no-progress-bar",
dest="useProgressBar",
help="Do not use curses based progress bar",
action="store_false",
default=True)
parser.add_option_group(group)
group = OptionGroup(parser, "Test Execution")
group.add_option("",
"--path",
dest="path",
help="Additional paths to add to testing environment",
action="append",
type=str,
default=[])
group.add_option("",
"--vg",
dest="useValgrind",
help="Run tests under valgrind",
action="store_true",
default=False)
group.add_option("",
"--vg-leak",
dest="valgrindLeakCheck",
help="Check for memory leaks under valgrind",
action="store_true",
default=False)
group.add_option("",
"--vg-arg",
dest="valgrindArgs",
metavar="ARG",
help="Specify an extra argument for valgrind",
type=str,
action="append",
default=[])
group.add_option("",
"--time-tests",
dest="timeTests",
help="Track elapsed wall time for each test",
action="store_true",
default=False)
group.add_option("",
"--no-execute",
dest="noExecute",
help="Don't execute any tests (assume PASS)",
action="store_true",
default=False)
parser.add_option_group(group)
group = OptionGroup(parser, "Test Selection")
group.add_option("",
"--max-tests",
dest="maxTests",
metavar="N",
help="Maximum number of tests to run",
action="store",
type=int,
default=None)
group.add_option("",
"--max-time",
dest="maxTime",
metavar="N",
help="Maximum time to spend testing (in seconds)",
action="store",
type=float,
default=None)
group.add_option("",
"--shuffle",
dest="shuffle",
help="Run tests in random order",
action="store_true",
default=False)
group.add_option("",
"--filter",
dest="filter",
metavar="REGEX",
help=("Only run tests with paths matching the given "
"regular expression"),
action="store",
default=None)
parser.add_option_group(group)
group = OptionGroup(parser, "Debug and Experimental Options")
group.add_option("",
"--debug",
dest="debug",
help="Enable debugging (for 'lit' development)",
action="store_true",
default=False)
group.add_option("",
"--show-suites",
dest="showSuites",
help="Show discovered test suites",
action="store_true",
default=False)
group.add_option("",
"--repeat",
dest="repeatTests",
metavar="N",
help="Repeat tests N times (for timing)",
action="store",
default=None,
type=int)
parser.add_option_group(group)
(opts, args) = parser.parse_args()
if not args:
parser.error('No inputs specified')
if opts.numThreads is None:
# Python <2.5 has a race condition causing lit to always fail with numThreads>1
# http://bugs.python.org/issue1731717
# I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
# threads by default there.
if sys.hexversion >= 0x2050200:
opts.numThreads = Util.detectCPUs()
else:
opts.numThreads = 1
inputs = args
# Create the user defined parameters.
userParams = dict(builtinParameters)
for entry in opts.userParameters:
if '=' not in entry:
name, val = entry, ''
else:
name, val = entry.split('=', 1)
userParams[name] = val
# Create the global config object.
litConfig = LitConfig.LitConfig(progname=os.path.basename(sys.argv[0]),
path=opts.path,
quiet=opts.quiet,
useValgrind=opts.useValgrind,
valgrindLeakCheck=opts.valgrindLeakCheck,
valgrindArgs=opts.valgrindArgs,
noExecute=opts.noExecute,
ignoreStdErr=False,
debug=opts.debug,
isWindows=(platform.system() == 'Windows'),
params=userParams,
config_prefix=opts.configPrefix)
tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
if opts.showSuites:
suitesAndTests = {}
for t in tests:
if t.suite not in suitesAndTests:
suitesAndTests[t.suite] = []
suitesAndTests[t.suite].append(t)
print '-- Test Suites --'
suitesAndTests = suitesAndTests.items()
suitesAndTests.sort(key=lambda (ts, _): ts.name)
for ts, ts_tests in suitesAndTests:
print ' %s - %d tests' % (ts.name, len(ts_tests))
print ' Source Root: %s' % ts.source_root
print ' Exec Root : %s' % ts.exec_root
# Select and order the tests.
numTotalTests = len(tests)
# First, select based on the filter expression if given.
if opts.filter:
try:
rex = re.compile(opts.filter)
except:
parser.error("invalid regular expression for --filter: %r" %
(opts.filter))
tests = [t for t in tests if rex.search(t.getFullName())]
# Then select the order.
if opts.shuffle:
random.shuffle(tests)
else:
tests.sort(key=lambda t: t.getFullName())
# Finally limit the number of tests, if desired.
if opts.maxTests is not None:
tests = tests[:opts.maxTests]
# Don't create more threads than tests.
opts.numThreads = min(len(tests), opts.numThreads)
extra = ''
if len(tests) != numTotalTests:
extra = ' of %d' % numTotalTests
header = '-- Testing: %d%s tests, %d threads --' % (len(tests), extra,
opts.numThreads)
if opts.repeatTests:
tests = [
t.copyWithIndex(i) for t in tests for i in range(opts.repeatTests)
]
progressBar = None
if not opts.quiet:
if opts.succinct and opts.useProgressBar:
try:
tc = ProgressBar.TerminalController()
progressBar = ProgressBar.ProgressBar(tc, header)
except ValueError:
print header
progressBar = ProgressBar.SimpleProgressBar('Testing: ')
else:
print header
startTime = time.time()
display = TestingProgressDisplay(opts, len(tests), progressBar)
provider = TestProvider(tests, opts.maxTime)
try:
import win32api
except ImportError:
pass
else:
def console_ctrl_handler(type):
provider.cancel()
return True
win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
runTests(opts.numThreads, litConfig, provider, display)
display.finish()
if not opts.quiet:
print 'Testing Time: %.2fs' % (time.time() - startTime)
# Update results for any tests which weren't run.
for t in tests:
if t.result is None:
t.setResult(Test.UNRESOLVED, '', 0.0)
# List test results organized by kind.
hasFailures = False
byCode = {}
for t in tests:
if t.result not in byCode:
byCode[t.result] = []
byCode[t.result].append(t)
if t.result.isFailure:
hasFailures = True
# FIXME: Show unresolved and (optionally) unsupported tests.
for title, code in (('Unexpected Passing Tests', Test.XPASS),
('Failing Tests', Test.FAIL)):
elts = byCode.get(code)
if not elts:
continue
print '*' * 20
print '%s (%d):' % (title, len(elts))
for t in elts:
print ' %s' % t.getFullName()
print
if opts.timeTests:
# Collate, in case we repeated tests.
times = {}
for t in tests:
key = t.getFullName()
times[key] = times.get(key, 0.) + t.elapsed
byTime = list(times.items())
byTime.sort(key=lambda (name, elapsed): elapsed)
if byTime:
Util.printHistogram(byTime, title='Tests')
for name, code in (
('Expected Passes ', Test.PASS),
('Expected Failures ', Test.XFAIL),
('Unsupported Tests ', Test.UNSUPPORTED),
('Unresolved Tests ', Test.UNRESOLVED),
('Unexpected Passes ', Test.XPASS),
('Unexpected Failures', Test.FAIL),
):
if opts.quiet and not code.isFailure:
continue
N = len(byCode.get(code, []))
if N:
print ' %s: %d' % (name, N)
# If we encountered any additional errors, exit abnormally.
if litConfig.numErrors:
print >> sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
sys.exit(2)
# Warn about warnings.
if litConfig.numWarnings:
print >> sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings
if hasFailures:
sys.exit(1)
sys.exit(0)
def ProxyDaemon(log_file='/var/log/ssl_proxy_access.log',
HandlerClass=ProxyRequestHandler,
ServerClass=ProxyServer,
protocol="HTTP/1.0",
thread=True,
interactive=False,
port=1443,
spoof_ssl_config={'DEFAULT': True},
display_filter=None,
cert=None,
dump_payload=False,
http_redirect_table={},
ssl_redirect_table={},
log=logging):
sys.setcheckinterval(10)
server_address = ('0.0.0.0', port)
HandlerClass.protocol_version = protocol
HandlerClass.cert = cert
HandlerClass.spoof_ssl_config = spoof_ssl_config
HandlerClass.log = log
# Example for http_redirect_table:
# http_redirect_table == {'https://host.example.com:443/(foo/.*)':
# 'http://otherhost.example.com:7433/\1'}
# This will match URLs like https://host.example.com:443/foo/bar
# And rewrite them as http://otherhost.example.com:7433/foo/bar
# Before making connections.
HandlerClass.http_redirect_table = http_redirect_table
# Example for ssl_redirect_table:
# ssl_redirect_table == {('orig.example.com', 443):
# ('new_dest.example.com', 443)}
HandlerClass.ssl_redirect_table = ssl_redirect_table
HandlerClass.dump_payload = dump_payload
if interactive:
HandlerClass.interactive = True
HandlerClass.display_filter = display_filter
log_fh = None
else:
try:
log_fh = open(log_file, 'a')
except:
log.fatal('Could not open access log: %s' % log_file)
sys.exit(os.EX_CANTCREAT)
HandlerClass.log_file = log_fh
HandlerClass.interactive = False
if thread:
reusable_socketserver = SocketServer.ThreadingTCPServer
else:
reusable_socketserver = SocketServer.TCPServer
reusable_socketserver.allow_reuse_address = True
reusable_socketserver.daemon_threads = True
try:
httpd = reusable_socketserver(server_address, HandlerClass)
except socket.error:
log.exception('Could not bind to %s:%s' % server_address)
sys.exit(os.EX_OSERR)
sa = httpd.socket.getsockname()
log.info('Listening on %s port %s ...' % (sa[0], sa[1]))
# Need to get the logfile closed before the shutdown happens.
try:
httpd.serve_forever()
except KeyboardInterrupt:
log.info('Shutting down from keyboard...')
if log_fh:
log_fh.close()
except:
if log_fh:
log_fh.close()
log.exception('Exception after serve_forever')
raise