def profile():
"""
Code profiler using YAPPI
http://code.google.com/p/yappi
"""
import yappi
yappi.start()
start(False)
yappi.stop()
for pstat in yappi.get_stats(yappi.SORTTYPE_TSUB):
print pstat
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
#fixup python include path, so we can include other project directories
sys.path.append(os.path.join(os.getcwd(), "../"))
import yappi
from client.start import main
yappi.start(True)
try:
main()
except:
pass
for it in yappi.get_stats(yappi.SORTTYPE_TSUB,
yappi.SORTORDER_DESCENDING,
yappi.SHOW_ALL):
print it
yappi.stop()
import time
import yappi
from threading import Thread
NTHREAD = 5
MAXRDEPTH = 10
def foo2():
time.sleep(0.1)
def foo(rdepth):
if rdepth == MAXRDEPTH:
return
foo2()
foo(rdepth + 1)
if __name__ == "__main__":
yappi.start(True) # start profiler with built-in profiling set to True
for i in range(NTHREAD): # multiple threads calling a recursive function.
thr = Thread(foo(1))
thr.start()
thr.join()
for it in yappi.get_stats(yappi.SORTTYPE_TTOTAL, yappi.SORTORDER_ASCENDING, yappi.SHOW_ALL):
print it
def get_stats(self, *args, **kwrds):
return yappi.get_stats(*args, **kwrds)
def get_other_yappi_stats():
stats = yappi.get_stats(limit=0)
THREAD_HEADER = '\n\nname tid fname scnt ttot'
thread_index = stats.index(THREAD_HEADER)
stats[thread_index] = stats[thread_index].strip()
return stats[thread_index:]
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
#fixup python include path, so we can include other project directories
sys.path.append(os.path.join(os.getcwd(), "../"))
import yappi
from client.start import main
yappi.start(True)
try:
main()
except:
pass
for it in yappi.get_stats(yappi.SORTTYPE_TSUB, yappi.SORTORDER_DESCENDING,
yappi.SHOW_ALL):
print it
yappi.stop()
#Requires yappi to be installed, use easy_install yappi
import yappi
from tribler import run
from time import time
if __name__ == '__main__':
t1 = time()
yappi.start()
run()
yappi.stop()
print "YAPPI:", yappi.clock_type(), "tribler has run for", time() - t1, "seconds"
stats = yappi.get_stats(yappi.SORTTYPE_TSUB)
for func_stats in stats.func_stats[:50]:
print "YAPPI: %10dx %10.3fs" % (func_stats.ncall, func_stats.tsub), func_stats.name
#yappi.print_stats(yappi.SORTTYPE_TTOTAL)
import yappi
import cProfile
def a_very_long_function_name():
pass
def state(entry):
print entry
def foo():
time.sleep(1)
cProfile.run('foo()', 'fooprof')
import pstats
p = pstats.Stats('fooprof')
p.strip_dirs().sort_stats(-1).print_stats()
yappi.start(True)
foo()
a_very_long_function_name()
yappi.stop()
yappi.enum_stats(state)
li = yappi.get_stats(1, 1, 8)
for it in li:
print it
yappi.clear_stats()
def main():
""" Main execution. """
# Begin timing execution
starttime = time.time()
# Check that RNAhybrid exists, and is executable
assert(os.path.exists(RNAHYBRID_PATH) and os.access(RNAHYBRID_PATH, os.X_OK))
# Same for Perl filter_rnahybrid script
assert(os.path.exists(FILTER_PATH) and os.access(FILTER_PATH, os.X_OK))
assert(os.path.exists(FILTER_SETTINGS))
assert(os.path.exists(ALIGN_PATH))
assert(os.path.exists(LOG_PATH))
usage = "usage: %prog [OPTIONS] outfile"
parser = OptionParser(usage)
# This ignores hyperthreading pseudo-cores, which is fine since we hose the ALU
parser.add_option("-j", help="Threads. We parallelize the invocations of RNAhybrid. [Default: # of CPU cores]",
default=os.sysconf('SC_NPROCESSORS_ONLN'), action="store", type="int", dest="threads")
parser.add_option("-p", help="Profile. Invokes the yappi python profiling engine. Will slow down execution.",
default=False, action="store_true", dest="profileMe")
group = OptionGroup(parser, "Range Settings (optional)")
group.add_option("--start-num", help="What number miRNA ortholog group to start scanning from (inclusive).",
default=-1, action="store", type="int", dest="startNum")
group.add_option("--stop-num", help="What number miRNA ortholog group to STOP scanning at (exclusive).",
default=-1, action="store", type="int", dest="stopNum")
parser.add_option_group(group)
(options, args) = parser.parse_args()
# Sanity check range inputs
range_given = False
if options.startNum >= 0 or options.stopNum >= 0:
if not (options.startNum >= 0 and options.stopNum >= 0):
parser.error("If you specifiy a start/stop, you must specify both ends of the range!")
if options.startNum >= options.stopNum:
parser.error("Invalid scan range.")
if options.startNum == 1:
print "WARNING: This is a Python range, where lists are zero indexed! Are you sure you mean '1'?"
range_given = True
if len(args) == 0 and not range_given:
parser.error("You must specify an outfile if you don't provide range options.\n\nTry -h for help.")
# Set logging output, as flags passed were valid.
# We log DEBUG and higher to log file, and write INFO and higher to console.
datestamp = datetime.datetime.now().strftime("%m%d-%H%M")
logfile = LOG_PATH + datestamp + '.' + socket.gethostname().split('.')[0]
logging.basicConfig(filename = logfile, filemode = 'w',
format = '%(asctime)s: %(name)-25s: %(levelname)-8s: %(message)s',
level = logging.DEBUG)
# define a Handler which writes INFO messages or higher to the sys.stderr
console = logging.StreamHandler()
console.setLevel(logging.INFO)
# set a format which is simpler for console use
formatter = logging.Formatter('%(name)-25s: %(levelname)-8s: %(message)s')
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
log_main = logging.getLogger('main')
log_main.info('Logging to %s' % logfile)
log_main.info('Starting run on %s' % socket.gethostname().split('.')[0])
log_main.info('miRNA range: [%d, %d]' % (options.startNum, options.stopNum))
if options.profileMe:
log_main.info('Profiling enabled.')
yappi.start()
# Connect to Database
oracle_password = open(ORACLE_PWFILE, 'r').readline()
dbase = cx_Oracle.connect(ORACLE_USERNAME, oracle_password, ORACLE_SERVER + ':1521/' + ORACLE_DB)
log_main.info("Connected to [read] %s (USER: %s)" % (dbase.dsn, ORACLE_USERNAME))
# Connect to writing database, too
oracle_write_password = open(ORACLE_WRITE_PWFILE, 'r').readline()
dbase_write = cx_Oracle.connect(ORACLE_WRITE_USERNAME, oracle_write_password,
ORACLE_WRITE_SERVER + ':1521/' + ORACLE_WRITE_DB)
log_main.info("Connected to [write] %s (USER: %s)" % (dbase_write.dsn, ORACLE_WRITE_USERNAME))
# Define the species sets we're looking at
# Global TAXIDs. Local <-> Global <-> Org is available in CAFEUSER.LOCALTAXID_TO_ORG
globalTaxMap = {'Hs': 9606, # Human
'Cf': 9615, # Dog
'Rn': 10116, # Rat
'Gg': 9031, # Chicken
'Mm': 10094} # Mouse
revGlobalTaxMap = dict((v,k) for k, v in globalTaxMap.iteritems())
localTaxMap = {'Hs': 7951, # Human
'Cf': 7959, # Dog
'Rn': 8385, # Rat
'Gg': 7458, # Chicken
'Mm': 8364} # Mouse
revLocalTaxMap = dict((v,k) for k, v in localTaxMap.iteritems())
UCSCTaxMap = {'hg18': 'Hs', # Human
'canFam2': 'Cf', # Dog
'rn4': 'Rn', # Rat
'galGal3': 'Gg', # Chicken
'mm9': 'Mm'} # Mouse
revUCSCTaxMap = dict((v,k) for k, v in UCSCTaxMap.iteritems())
shortToMAF = {'Hs': 'Human',
'Cf': 'Dog',
'Rn': 'Rat',
'Gg': 'Chicken',
'Mm': 'Mouse'}
revShortToMAF = dict((v,k) for k, v in shortToMAF.iteritems())
assert(len(globalTaxMap) == len(localTaxMap))
assert(len(localTaxMap) == len(UCSCTaxMap))
# Species Index for determining weights later
# We lookup weights using: index = sum(2**species_index)
species_index = {'hg18': 0,
'panTro2': 1,
'ponAbe2': 2,
'mm9': 3,
'rn4': 4,
'rheMac2': 5,
'monDom4': 6,
'bosTau4': 7,
'canFam2': 8,
'equCab2': 9}
# Import our huge weight "matrix", which is a long list
weight_matrix = scan_data.human_weight
### ---------------------------------------------
# First, we get the microRNA clusters.
mirna_queries = _get_microrna_data(dbase, range_given, options.startNum, options.stopNum, localTaxMap, UCSCTaxMap)
### ---------------------------------------------
# Then, we get everything necessary for mRNA data.
# Check out these functions for a description of these data structures.
homologene_to_mrna = _get_homologene_to_mrna(dbase, globalTaxMap)
mrna_to_seq = _get_mrna_to_seq(dbase, globalTaxMap)
mrna_to_exons = _get_mrna_to_exons(dbase, globalTaxMap)
sanity_overlap_check(mrna_to_exons)
# Coding sequence excludes 3' and 5'
#mrna_to_cds = _get_mrna_to_cds(dbase, globalTaxMap)
#sanity_overlap_check(mrna_to_cds)
### ---------------------------------------------
### We do this by getting entire gene positions from PAP.GENE_COORDINATES, then
### selecting only exons using PAP.MRNA_COORDINATES.
### ---------------------------------------------
### First, start looping to populate our work_queue for threads to work
### Then, when filled, keep topping it off, while we periodically poll result_queue
### ---------------------------------------------
work_queue_size = 500
low_water_mark = 400 # Size at which we refill the work_queue
fillup_increment = 50 # and how much to fill it by
critical_low_mark = 100 # Size at which we stall threads, as work queue is too small!
result_queue_size = 500
high_water_mark = 100 # Size at which we send off data to collector (or write to disk)
drain_increment = 50 # how many results to send off/write at a given time
critical_high_mark = 400 # Site at which we stall threads, as result_queue is too big!
assert(low_water_mark < work_queue_size)
assert(critical_low_mark < low_water_mark)
assert(high_water_mark < result_queue_size)
assert(high_water_mark < critical_high_mark)
assert(fillup_increment < (work_queue_size - low_water_mark))
assert(drain_increment < (result_queue_size - high_water_mark))
_out_of_work = False
work_queue = Queue.Queue(maxsize = work_queue_size) # A threadsafe producer/consumer Queue
result_queue = Queue.Queue(maxsize = result_queue_size) # Same, but for product of RNAhybrid
_first_run = True
_work_queue_generator = _get_item_for_work_queue(dbase, mirna_queries, homologene_to_mrna, mrna_to_seq, mrna_to_exons)
lol = []
for _ in range(100):
lol.append(_work_queue_generator.next())
# WARNING: Queue calls can and /will/ block! Be careful!
while True:
log_main.info("(work|result|threads) = %s, %s, %s " %
(str(work_queue.qsize()), str(result_queue.qsize()), threading.active_count()-1))
time.sleep(2)
while (work_queue.qsize() < low_water_mark) and (_out_of_work == False):
log_main.debug("Filling work queue.")
# We call a generator to keep our work_queue mostly full
for _ in range(fillup_increment):
try:
work_queue.put(lol.pop())
# work_queue.put(_work_queue_generator.next())
except StopIteration:
log_main.debug("Out of work: Waiting for work_queue to be emptied by threads ...")
_out_of_work = True
except IndexError:
log_main.debug("Out of work: Waiting for work_queue to be emptied by threads ...")
_out_of_work = True
break
while result_queue.qsize() > high_water_mark:
log_main.debug("Draining result_queue.")
# These queue checks are imperfect, but should never be triggered.
if result_queue.qsize() > critical_high_mark:
log_main.error("Result queue is too big! Something is wrong!")
consume_event.clear() # Stall threads.
if (work_queue.qsize() < critical_low_mark) and (_out_of_work == False):
log_main.error("Work queue is too small! Something is wrong!")
consume_event.clear() # Stall threads.
# Insert up to drain_increment elements into the database
_write_results_to_db(dbase_write, result_queue, drain_increment)
if _first_run:
# First time we're running, so spawn threads
log_main.info("Spawning %s threads." % str(options.threads))
consume_event = threading.Event()
for i in range(options.threads):
thread = RNAHybridThread(i+1, consume_event, work_queue, result_queue, species_index, weight_matrix, mrna_to_seq, mrna_to_exons)
thread.daemon = True
thread.start()
_first_run = False
consume_event.set() # Let's go to work!
if _out_of_work:
# First, wait for results_queue to be completely emptied
while (threading.active_count() - 1) > 0:
_write_results_to_db(dbase_write, result_queue, 10)
# At this point, all our threads have died.
time.sleep(1) # A second to catch up, just in case.
while result_queue.qsize() > 0:
_write_results_to_db(dbase_write, result_queue, 1)
log_main.info("All done!")
break
consume_event.set() # Let's go to work!
log_main.info("Execution took: %s secs." % (time.time()-starttime))
# Print output of profiling, if it was enabled.
if options.profileMe:
log_main.debug('Profiling data:')
stats = yappi.get_stats()
for stat in stats:
log_main.debug(stat)
yappi.stop()
import time
import yappi
from threading import Thread
NTHREAD = 5
MAXRDEPTH = 10
def foo2():
time.sleep(0.1)
def foo(rdepth):
if (rdepth == MAXRDEPTH):
return
foo2()
foo(rdepth + 1)
if __name__ == "__main__":
yappi.start(True) # start profiler with built-in profiling set to True
for i in range(NTHREAD): # multiple threads calling a recursive function.
thr = Thread(foo(1))
thr.start()
thr.join()
for it in yappi.get_stats(yappi.SORTTYPE_TTOTAL, yappi.SORTORDER_ASCENDING,
yappi.SHOW_ALL):
print it
def get_config(self):
config = {}
config['io_drivers'] = self.io_driver_list.get_config()
config['viewers'] = self.viewers.get_config()
return config
def set_config(self, config):
if 'io_drivers' in config:
self.io_driver_list.set_config(config['io_drivers'])
if 'viewers' in config:
self.viewers.set_config(config['viewers'])
self.variables.clear()
vs = Variables()
viewers = Viewers(variables = vs)
iodl = IODriverList(variables = vs, viewers_instance = viewers)
proj = PlotOMatic(io_driver_list = iodl, variables = vs, viewers = viewers)
proj.start()
proj.configure_traits()
proj.stop()
if PROFILE:
print "Generating Statistics"
yappi.stop()
stats = yappi.get_stats(yappi.SORTTYPE_TSUB, yappi.SORTORDER_DESCENDING, 300) #yappi.SHOW_ALL)
for stat in stats:
print stat
config = {}
config['io_drivers'] = self.io_driver_list.get_config()
config['viewers'] = self.viewers.get_config()
return config
def set_config(self, config):
if 'io_drivers' in config:
self.io_driver_list.set_config(config['io_drivers'])
if 'viewers' in config:
self.viewers.set_config(config['viewers'])
self.variables.clear()
vs = Variables()
viewers = Viewers(variables=vs)
iodl = IODriverList(variables=vs, viewers_instance=viewers)
proj = PlotOMatic(io_driver_list=iodl, variables=vs, viewers=viewers)
proj.start()
proj.configure_traits()
proj.stop()
if PROFILE:
print "Generating Statistics"
yappi.stop()
stats = yappi.get_stats(yappi.SORTTYPE_TSUB, yappi.SORTORDER_DESCENDING,
300) #yappi.SHOW_ALL)
for stat in stats:
print stat
def run(self):
running = True
# self.tracker = pickle.load(open("tracker-5pt-halfdata.dat", "rb"))
self.tracker = pickle.load(open("model861.dat", "rb"))
yappi.start(True)
while running:
time.sleep(0.01)
while self.childConn.poll(0):
ev = self.childConn.recv()
if ev[0] == "quit":
running = False
if ev[0] == "frame":
self.currentFrame = ev[1]
self.trackingPending = True
self.currentFrameNum = ev[2]
if ev[0] == "faces":
posModel = []
if len(ev[1]) == 0:
continue
face = ev[1][0]
w = face[2] - face[0]
h = face[3] - face[1]
for pt in self.detectPtsPos:
posModel.append((face[0] + pt[0] * w, face[1] + pt[1] * h))
if self.currentModel is None:
self.currentModel = posModel
if ev[0] == "reinit":
self.currentModel = None
time.sleep(0.01)
if self.trackingPending and self.currentModel is not None:
# Normalise input image using procrustes
self.normIm = normalisedImageOpt.NormalisedImage(
self.currentFrame, self.currentModel, self.meanFace, {}
)
# normalisedImage.SaveNormalisedImageToFile(self.normIm, "img.jpg")
# Convert coordinates to normalised space
normPosModel = [self.normIm.GetNormPos(*pt) for pt in self.currentModel]
# print "normPosModel",normPosModel
# Initialise prev features if necessary
if self.prevFrameFeatures is None:
self.prevFrameFeatures = self.tracker.CalcPrevFrameFeatures(self.normIm, normPosModel)
# Make prediction
pred = self.tracker.Predict(self.normIm, normPosModel, self.prevFrameFeatures)
# Convert prediction back to image space
self.currentModel = [self.normIm.GetPixelPosImPos(*pt) for pt in pred]
# print self.currentFrameNum, self.currentModel
# io.imsave("currentFrame.jpg", self.currentFrame)
self.childConn.send(["tracking", self.currentModel])
self.trackingPending = False
self.prevFrameFeatures = self.tracker.CalcPrevFrameFeatures(self.normIm, pred)
self.count += 1
if self.count > 10:
yappi.stop()
stats = yappi.get_stats()
pickle.dump(stats, open("prof.dat", "wb"), protocol=-1)
self.count = 0
yappi.start(True)
if self.trackingPending and self.currentModel is None:
self.childConn.send(["tracking", None])
self.trackingPending = False
self.childConn.send(["done", 1])
def recursive(n=100):
if n == 0:
return n
return n+recursive(n-1)
yappi.start()
recursive(100)
yappi.stop()
yappi.start()
class TestThread(Thread):
def __init__(self, n=100):
Thread.__init__(self)
self.n = n
def run(self):
self.output = sum(x for x in xrange(self.n))
t_list = []
for t_n in xrange(10):
t = TestThread(100000)
t_list.append(t)
t.start()
for t in t_list:
t.join()
print t.output
yappi.stop()
print('\n'.join(yappi.get_stats()))
def get_stats(self, *args, **kwrds):
return yappi.get_stats(*args, **kwrds)
import yappi
import cProfile
def a_very_long_function_name():
pass
def state(entry):
print entry
def foo():
time.sleep(1)
cProfile.run('foo()', 'fooprof')
import pstats
p = pstats.Stats('fooprof')
p.strip_dirs().sort_stats(-1).print_stats()
yappi.start(True)
foo()
a_very_long_function_name()
yappi.stop()
yappi.enum_stats(state)
li = yappi.get_stats(1, 1, 8)
for it in li:
print it
yappi.clear_stats()
keys.sort()
working_dir = "analyse-modeling-%s" % time.strftime("%Y%m%d-%H%M%S")
base_dir = os.getcwd()
os.makedirs(working_dir)
os.chdir(working_dir)
for run in keys:
xsd = XSDataInputSaxsAnalysisModeling(scatterCurve=XSDataFile(XSDataString(run)), graphFormat=XSDataString("png"))
reprocess.startJob(xsd)
print("All %i jobs queued after %.3fs" % (len(args), time.time() - reprocess.startTime))
reprocess.join()
if yappi: yappi.stop()
print("All %i jobs processed after %.3fs" % (len(args), time.time() - reprocess.startTime))
print reprocess.statistics()
if yappi:
stat = yappi.get_stats(sort_type=yappi.SORTTYPE_TTOT)
res = {}
for i in stat.func_stats:
if i[0] in res:
res[i[0]][0] += i[1]
res[i[0]][1] += i[2]
else:
res[i[0]] = [i[1], i[2]]
keys = res.keys()
keys.sort(sortn)
with open("yappi.out", "w") as f:
f.write("ncall\t\ttotal\t\tpercall\t\tfunction%s" % (os.linesep))
for i in keys:
f.write("%8s\t%16s\t%16s\t%s%s" % (res[i][0], res[i][1], res[i][1] / res[i][0], i, os.linesep))
print("Profiling information written in yappi.out")
import yappi
yappi.start(True)
f = open("foo", "w")
f.write("bar")
f.close()
yappi.stop()
stats = yappi.get_stats(yappi.SORTTYPE_NAME)
for stat in stats:
print stat