def test_raw_forced_delayed(self): comments = [] class Test_JavaAbortFinalizable(Object): def __init__(self, name, toAbort): self.name = name self.toAbort = toAbort def __repr__(self): return "<" + self.name + ">" def finalize(self): gc.notifyPreFinalization() comments.append("del " + self.name) gc.abortDelayedFinalization(self.toAbort) # We manually restore weak references: gc.restoreWeakReferences(self.toAbort) gc.notifyPostFinalization() class Test_Finalizable(object): def __init__(self, name): self.name = name def __repr__(self): return "<" + self.name + ">" def __del__(self): comments.append("del " + self.name) def callback_a(obj): comments.append("callback_a") def callback_b(obj): comments.append("callback_b") a = Test_Finalizable("a") wa = weakref.ref(a, callback_a) b = Test_JavaAbortFinalizable("b", a) wb = weakref.ref(b, callback_b) gc.addJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION) gc.addJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS) self.assertTrue(gc.delayedFinalizationEnabled()) self.assertTrue(gc.delayedWeakrefCallbacksEnabled()) self.assertEqual(len(comments), 0) del a del b System.gc() time.sleep(2) self.assertIsNotNone(wa()) self.assertIsNone(wb()) self.assertIn('del b', comments) self.assertNotIn('callback_a', comments) self.assertIn('callback_b', comments) self.assertNotIn('del a', comments) self.assertEqual(2, len(comments)) gc.removeJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION) gc.removeJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS)
def test_raw_forced_delayed(self): comments = [] class Test_JavaAbortFinalizable(Object): def __init__(self, name, toAbort): self.name = name self.toAbort = toAbort def __repr__(self): return "<"+self.name+">" def finalize(self): gc.notifyPreFinalization() comments.append("del "+self.name) gc.abortDelayedFinalization(self.toAbort) # We manually restore weak references: gc.restoreWeakReferences(self.toAbort) gc.notifyPostFinalization() class Test_Finalizable(object): def __init__(self, name): self.name = name def __repr__(self): return "<"+self.name+">" def __del__(self): comments.append("del "+self.name) def callback_a(obj): comments.append("callback_a") def callback_b(obj): comments.append("callback_b") a = Test_Finalizable("a") wa = weakref.ref(a, callback_a) b = Test_JavaAbortFinalizable("b", a) wb = weakref.ref(b, callback_b) gc.addJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION) gc.addJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS) self.assertTrue(gc.delayedFinalizationEnabled()) self.assertTrue(gc.delayedWeakrefCallbacksEnabled()) self.assertEqual(len(comments), 0) del a del b System.gc() time.sleep(2) self.assertIsNotNone(wa()) self.assertIsNone(wb()) self.assertIn('del b', comments) self.assertNotIn('callback_a', comments) self.assertIn('callback_b', comments) self.assertNotIn('del a', comments) self.assertEqual(2, len(comments)) gc.removeJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION) gc.removeJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS)
def test_raw_forced_delayedWeakrefCallback(self): comments = [] resurrected = [] class Test_JavaResurrectFinalizable(Object): def __init__(self, name, toResurrect): self.name = name self.toResurrect = toResurrect def __repr__(self): return "<"+self.name+">" # Note that this type of finalizer is usually not recommended # as it gets lost in case of resurrection. def finalize(self): gc.notifyPreFinalization() comments.append("del "+self.name) resurrected.append(self.toResurrect) # We manually restore weak references: gc.restoreWeakReferences(self.toResurrect) gc.notifyPostFinalization() class Test_Finalizable(object): def __init__(self, name): self.name = name def __repr__(self): return "<"+self.name+">" def __del__(self): comments.append("del "+self.name) def callback(obj): comments.append("callback") a = Test_Finalizable("a") b = Test_JavaResurrectFinalizable("b", a) wa = weakref.ref(a, callback) gc.removeJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION) gc.addJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS) self.assertFalse(gc.delayedFinalizationEnabled()) self.assertTrue(gc.delayedWeakrefCallbacksEnabled()) self.assertEqual(len(comments), 0) aStr = str(a) del a del b System.gc() time.sleep(2) self.assertIn("del a", comments) self.assertIn("del b", comments) self.assertEqual(1, len(resurrected)) self.assertEqual(str(resurrected[0]), aStr) self.assertIsNotNone(wa()) self.assertEqual(resurrected[0], wa()) self.assertNotIn("callback", comments) self.assertEqual(2, len(comments)) gc.removeJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS)
def test_raw_forced_delayedWeakrefCallback(self): comments = [] resurrected = [] class Test_JavaResurrectFinalizable(Object): def __init__(self, name, toResurrect): self.name = name self.toResurrect = toResurrect def __repr__(self): return "<" + self.name + ">" # Note that this type of finalizer is usually not recommended # as it gets lost in case of resurrection. def finalize(self): gc.notifyPreFinalization() comments.append("del " + self.name) resurrected.append(self.toResurrect) # We manually restore weak references: gc.restoreWeakReferences(self.toResurrect) gc.notifyPostFinalization() class Test_Finalizable(object): def __init__(self, name): self.name = name def __repr__(self): return "<" + self.name + ">" def __del__(self): comments.append("del " + self.name) def callback(obj): comments.append("callback") a = Test_Finalizable("a") b = Test_JavaResurrectFinalizable("b", a) wa = weakref.ref(a, callback) gc.removeJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION) gc.addJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS) self.assertFalse(gc.delayedFinalizationEnabled()) self.assertTrue(gc.delayedWeakrefCallbacksEnabled()) self.assertEqual(len(comments), 0) aStr = str(a) del a del b System.gc() time.sleep(2) self.assertIn("del a", comments) self.assertIn("del b", comments) self.assertEqual(1, len(resurrected)) self.assertEqual(str(resurrected[0]), aStr) self.assertIsNotNone(wa()) self.assertEqual(resurrected[0], wa()) self.assertNotIn("callback", comments) self.assertEqual(2, len(comments)) gc.removeJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS)
def runGCIfJython(): try: currentIndex = GCDetector.gcIndex gcCount = 0 detector = GCDetector() detector = None while currentIndex == GCDetector.gcIndex and gcCount < maxGCRun: System.gc() gcCount += 1 time.sleep(0.1) except: pass
def test_finalization_preprocess_and_postprocess(self): # Note that this test is done here again (already was in another class # in this module), to see that everything works as it should also with # a different flag-context. comments = [] self0 = self class A: def __del__(self): self0.assertIn("run PreProcess", comments) comments.append("A del") # let's simulate a time-consuming finalizer # to ensure that post finalization processing # is sensitive to this time.sleep(0.5) comments.append("A del done") class PreProcess(Runnable): def run(self): self0.assertEqual(comments, []) comments.append("run PreProcess") class PostProcess(Runnable): def run(self): self0.assertIn("run PreProcess", comments) self0.assertIn("A del", comments) self0.assertIn("A del done", comments) comments.append("run PostProcess") a = A() a = None prePr = PreProcess() postPr = PostProcess() time.sleep(1) # <- to avoid that the newly registered processes # become subject to previous run (remember: We # are not in monitor-mode, i.e. gc runs async. gc.registerPreFinalizationProcess(prePr) gc.registerPostFinalizationProcess(postPr) # Note that order matters here: # If the flag gc.DONT_FINALIZE_RESURRECTED_OBJECTS is used, # gc.registerPostFinalizationProcess(postPr, 0) would lead to failure, # because postPr asserts that a's finalizer already ran. Since # DONT_FINALIZE_RESURRECTED_OBJECTS also inserted a postprocess, # to perform delayed finalization, the 0-index would prepend postPr # before the process that actually runs the finalizers. System.gc() # we wait a bit longer here, since PostProcess runs asynchronous # and must wait for the finalizer of A time.sleep(2) self.assertIn("run PostProcess", comments) comments = [] gc.unregisterPreFinalizationProcess(prePr) gc.unregisterPostFinalizationProcess(postPr)
def test_finalization_preprocess_and_postprocess(self): #Note that this test is done here again (already was in another class #in this module), to see that everything works as it should also with #a different flag-context. comments = [] self0 = self class A: def __del__(self): self0.assertIn("run PreProcess", comments) comments.append("A del") #let's simulate a time-consuming finalizer #to ensure that post finalization processing #is sensitive to this time.sleep(0.5) comments.append("A del done") class PreProcess(Runnable): def run(self): self0.assertEqual(comments, []) comments.append("run PreProcess") class PostProcess(Runnable): def run(self): self0.assertIn("run PreProcess", comments) self0.assertIn("A del", comments) self0.assertIn("A del done", comments) comments.append("run PostProcess") a = A() a = None prePr = PreProcess() postPr = PostProcess() time.sleep(1) # <- to avoid that the newly registered processes # become subject to previous run (remember: We # are not in monitor-mode, i.e. gc runs async. gc.registerPreFinalizationProcess(prePr) gc.registerPostFinalizationProcess(postPr) #Note that order matters here: #If the flag gc.DONT_FINALIZE_RESURRECTED_OBJECTS is used, #gc.registerPostFinalizationProcess(postPr, 0) would lead to failure, #because postPr asserts that a's finalizer already ran. Since #DONT_FINALIZE_RESURRECTED_OBJECTS also inserted a postprocess, #to perform delayed finalization, the 0-index would prepend postPr #before the process that actually runs the finalizers. System.gc() #we wait a bit longer here, since PostProcess runs asynchronous #and must wait for the finalizer of A time.sleep(2) self.assertIn("run PostProcess", comments) comments = [] gc.unregisterPreFinalizationProcess(prePr) gc.unregisterPostFinalizationProcess(postPr)
def assertTriggered(test, expected): deadline = time() + DelTrigger.TIMEOUT while time() < deadline: System.gc() System.runFinalization() try: test.assertEqual(expected, DelTrigger.triggered) return except AssertionError: if not expected: raise sleep(DelTrigger.TIMEOUT / 10) test.fail("Not triggered after {} seconds".format(DelTrigger.TIMEOUT))
def test_weakref_after_resurrection_threadsafe(self): resurrect = [] comments = [] class Test_Finalizable(object): def __init__(self, name): self.name = name def __repr__(self): return "<"+self.name+">" def __del__(self): comments.append("del "+self.name) class Test_Resurrection(object): def __init__(self, name): self.name = name def __repr__(self): return "<"+self.name+">" def __del__(self): comments.append("del "+self.name) if hasattr(self, "toResurrect"): resurrect.append(self) a = Test_Finalizable("a") wa = weakref.ref(a) c = Test_Resurrection("c") c.toResurrect = a wc = weakref.ref(c) del a del c try: gc.addJythonGCFlags(gc.PRESERVE_WEAKREFS_ON_RESURRECTION) System.gc() # We intentionally don't wait here, but want to observe # the situation with gc unfinnished. Note that wa() preserves # its result right away, due to thread-safe implementation. # Technically, the weak reference breaks and is restored after # gc-run finishes. However wa() blocks until the referent is # restored or the deletion is confirmed. except Exception: pass self.assertEqual(comments, []) self.assertEqual(resurrect, []) while comments == [] or resurrect == []: self.assertEqual(str(wa()), '<a>') self.assertEqual(wc(), None) self.assertEqual(str(wa()), '<a>') self.assertEqual(wc(), None)
def test_weakref_after_resurrection_threadsafe(self): resurrect = [] comments = [] class Test_Finalizable(object): def __init__(self, name): self.name = name def __repr__(self): return "<" + self.name + ">" def __del__(self): comments.append("del " + self.name) class Test_Resurrection(object): def __init__(self, name): self.name = name def __repr__(self): return "<" + self.name + ">" def __del__(self): comments.append("del " + self.name) if hasattr(self, "toResurrect"): resurrect.append(self) a = Test_Finalizable("a") wa = weakref.ref(a) c = Test_Resurrection("c") c.toResurrect = a wc = weakref.ref(c) del a del c try: gc.addJythonGCFlags(gc.PRESERVE_WEAKREFS_ON_RESURRECTION) System.gc() # We intentionally don't wait here, but want to observe # the situation with gc unfinnished. Note that wa() preserves # its result right away, due to thread-safe implementation. # Technically, the weak reference breaks and is restored after # gc-run finishes. However wa() blocks until the referent is # restored or the deletion is confirmed. except Exception: pass self.assertEqual(comments, []) self.assertEqual(resurrect, []) while comments == [] or resurrect == []: self.assertEqual(str(wa()), '<a>') self.assertEqual(wc(), None) self.assertEqual(str(wa()), '<a>') self.assertEqual(wc(), None)
def populateMusicList(self): #If there are files if not MusicPlayer.isFolderEmpty(): walker = Files.walk(Paths.get('../resources/music')) #Ridiculous that we should have to do this. Nevertheless, here goes. class musicFilterPred(Predicate): #@Override def test(self, song): #No Symbolic Links if Files.isRegularFile(song, LinkOption.NOFOLLOW_LINKS): # MP3, WAV, AAC only, for now. file = String(song.toFile().toURI().toString()) ext = file.substring(file.length() - 4, file.length()) #.XYZ if ext not in ('.wav', '.WAV', '.mp3', '.MP3', '.aac', '.AAC'): return False else: #We are presuming that a file with the correct extension is actually #of the format specified by the extension and a valid file. #We will handle if is not, but for now, filter out the obvious bad eggs. return True from java.util.function import Function class musicMapPred(Function): #@Override def apply(self, song): return Media(song.toFile().toURI().toString()) #The Result music_list = walker.filter(musicFilterPred()).map( musicMapPred()).collect(Collectors.toList()) #Close Stream walker.close() walker = None #Nudge GC System.gc() #Retun valid Music Lists only if music_list.isEmpty(): return None else: return music_list #No files else: return None
def get_jython_free_mem(self): # Force garbage collection first. from java.lang import Runtime from java.lang.ref import WeakReference from java.lang import Object from java.lang import System obj = Object() ref = WeakReference(obj) obj = None while ref.get() is not None: System.gc() # Calculate approx. available memory. return Runtime.getRuntime().freeMemory()
def run4(): import DemoExtension JyNI.JyRefMonitor_setMemDebugFlags(1) l = ([0, "test"], ) l[0][0] = l DemoExtension.argCountToString(l) del l #l[0][1] = None print "Leaks before GC:" monitor.listLeaks() System.gc() time.sleep(2) print "Leaks after GC:" monitor.listLeaks()
def run4(): import DemoExtension JyNI.JyRefMonitor_setMemDebugFlags(1) l = ([0, "test"],) l[0][0] = l DemoExtension.argCountToString(l) del l #l[0][1] = None print "Leaks before GC:" monitor.listLeaks() System.gc() time.sleep(2) print "Leaks after GC:" monitor.listLeaks()
def test_forced_with_extern_NonPyObjectFinalizer_that_notifies_gc(self): comments = [] class A: def __init__(self, index): self.index = index def __del__(self): comments.append("A_del_" + str(self.index)) class PreProcess(Runnable): preCount = 0 def run(self): PreProcess.preCount += 1 class PostProcess(Runnable): postCount = 0 def run(self): PostProcess.postCount += 1 prePr = PreProcess() postPr = PostProcess() time.sleep(1) # <- to avoid that the newly registered processes # become subject to previous run (remember: We # are not in monitor-mode, i.e. gc runs async. gc.registerPreFinalizationProcess(prePr) gc.registerPostFinalizationProcess(postPr) for i in range(4): f = A(i) del f #NastyFinalizer would cause this test occasionally to fail externFinalizer = GCTestHelper.NotSoNastyFinalizer() del externFinalizer for i in range(4, 8): f = A(i) del f System.gc() # we wait a bit longer here, since PostProcess runs asynchronous # and must wait for the finalizer of A time.sleep(4) self.assertEqual(len(comments), 8) self.assertEqual(PreProcess.preCount, 1) self.assertEqual(PostProcess.postCount, 1) comments = [] gc.unregisterPreFinalizationProcess(prePr) gc.unregisterPostFinalizationProcess(postPr)
def test_with_extern_NonPyObjectFinalizer_that_notifies_gc(self): comments = [] class A: def __init__(self, index): self.index = index def __del__(self): comments.append("A_del_"+str(self.index)) class PreProcess(Runnable): preCount = 0 def run(self): PreProcess.preCount += 1 class PostProcess(Runnable): postCount = 0 def run(self): PostProcess.postCount += 1 prePr = PreProcess() postPr = PostProcess() time.sleep(1) # <- to avoid that the newly registered processes # become subject to previous run (remember: We # are not in monitor-mode, i.e. gc runs async. gc.registerPreFinalizationProcess(prePr) gc.registerPostFinalizationProcess(postPr) for i in range(4): f = A(i) del f #NastyFinalizer would cause this test occasionally to fail externFinalizer = GCTestHelper.NotSoNastyFinalizer() del externFinalizer for i in range(4, 8): f = A(i) del f System.gc() #we wait a bit longer here, since PostProcess runs asynchronous #and must wait for the finalizer of A time.sleep(4) self.assertEqual(len(comments), 8) self.assertEqual(PreProcess.preCount, 1) self.assertEqual(PostProcess.postCount, 1) comments = [] gc.unregisterPreFinalizationProcess(prePr) gc.unregisterPostFinalizationProcess(postPr)
def load_eol_ids(inpath, tax): System.gc() rt = Runtime.getRuntime() print '# Memory', rt.freeMemory()/(1024*1024), rt.totalMemory()/(1024*1024) page_to_nodes = {} node_to_pages = {} with open(inpath, 'r') as infile: print '| Processing EOL page ids file %s' % (inpath) reader = csv.reader(infile) row_count = 0 hits = 0 for row in reader: row_count += 1 [page_id, idspace, source_id] = row if row_count % 250000 == 0: print row_count, row qid = QualifiedId(idspace, source_id) node = tax.lookupQid(qid) if node != None: # and node.isPotentialOtu(): nodes = page_to_nodes.get(page_id) if nodes == None: nodes = [(node, qid)] page_to_nodes[page_id] = nodes elif not in_alist(node, nodes): nodes.append((node, qid)) pages = node_to_pages.get(node) if pages == None: pages = [page_id] node_to_pages[node] = pages elif not page_id in pages: pages.append(page_id) print '| OTT nodes having at least one EOL page: %s' % len(node_to_pages) print '| EOL page ids having at least one OTT node: %s' % len(page_to_nodes) # Sort page ids for each OTT node (will use smallest one) for node in node_to_pages: pages = node_to_pages[node] if len(pages) > 1: pages.sort(key=int) return (page_to_nodes, node_to_pages)
def main(): from sys import argv if len(argv) == 2: classToFiles = lambda x: classToFile(x.__name__) if argv[1] == 'all': # traverse throught all listed packages for package in [com, java, javax, org]: # sun print 'proceed package', package.__name__ packageToFile(package) classTraverse(package, classToFiles, packageToFile) System.gc() print 'writen ', len(classSaved), ' files' while len(classSaved) != 0: del classSaved[0] elif argv[1] == 'packages': for package in [com, java, javax, org]: print 'proceed package', package.__name__ packageToFile(package) classTraverse(package, lambda x: None, packageToFile) System.gc() elif argv[1] == 'css': toFile('class.css', classCss(2)) elif argv[1] == 'index': print 'creating index.html' toFile('index.html', frameSet()) lnk = '<a href="%s.html">%s</a>' packages = [com, java, javax, org] links = [lnk % (p.__name__, p.__name__) for p in packages] start = ['<h1>(JYTHON) CLASS</h1>'] + ['<ul>'] start += ul('ind0', links) start += ['</ul>'] html = join(start, (' ' * 4) + '\n') print 'creating start.html' toFile('start.html', baseHtml('javaDoc', html)) else: arg = eval(argv[1]) if type(arg) == PyJavaPackage: classTraverse(arg, classToFileS, packageToFile) elif type(arg) != PyJavaPackage: classToFile(argv[1]) else: usage() else: usage()
def execute(self): import herschel, os obsCont = fa.load(self.filename + "_ObsCont.fits") hk = fa.load(self.filename + "_HPPHK.fits")["hk"] pointing = fa.load(self.filename + "_Pointing.fits") orbitEphem = fa.load(self.filename + "_OrbitEphem.fits") timeCorr = fa.load(self.filename + "_TimeCorr.fits") calTree = getCalTree("FM") for band in ("blue", "green", "red"): try: frames = fa.load(self.filename + "_" + band + "_level0Frames.fits") except java.io.IOException: continue print "running findBlocks", Date() frames = findBlocks(frames) print "photFlagBadPixels", Date() frames = photFlagBadPixels(frames, calTree=calTree) myMask = _mkMyMask(band) badPix = frames.getMask("BADPIXELS") dims = badPix.dimensions for ipix in range(dims[0]): for jpix in range(dims[1]): if myMask[ipix, jpix] == True: badPix[ipix, jpix, :] = True frames.setMask("BADPIXELS", badPix) # This checks for ADC & CL saturation print "photFlagSaturation", Date() frames = photFlagSaturation(frames, calTree=calTree, hkdata=hk, check="full") print "photConvDigit2Volts", Date() frames = photConvDigit2Volts(frames, calTree=calTree) print "Convert chopper position to angle", Date() frames = convertChopper2Angle(frames, calTree=calTree) System.gc() print "Add pointing", Date() frames = photAddInstantPointing(frames, pointing, calTree=calTree, orbitEphem=orbitEphem) System.gc() # we shall never use the pointing and ephemris data again print "cleanPlateauFrames ; ", Date() frames = cleanPlateauFrames(frames, calTree=calTree) System.gc() # now save the file print "Write prepared frames to disk", Date() prepared_filename = os.path.join(self.path, os.path.basename(self.filename)) fa.save(prepared_filename + "_" + band + "_PreparedFrames.fits", frames) print
def execute(self): import herschel, os obsCont = fa.load(self.filename + '_ObsCont.fits') hk = fa.load(self.filename + '_HPPHK.fits')['hk'] pointing = fa.load(self.filename + '_Pointing.fits') orbitEphem = fa.load(self.filename + '_OrbitEphem.fits') timeCorr = fa.load(self.filename + '_TimeCorr.fits') calTree = getCalTree("FM") for band in ('blue', 'green', 'red'): try: frames = fa.load(self.filename + '_' + band + '_level0Frames.fits') except java.io.IOException: continue print "running findBlocks", Date() frames = findBlocks(frames) print "photFlagBadPixels", Date() frames = photFlagBadPixels(frames, calTree=calTree) myMask = _mkMyMask(band) badPix = frames.getMask('BADPIXELS') dims = badPix.dimensions for ipix in range(dims[0]): for jpix in range(dims[1]): if (myMask[ipix,jpix] == True): badPix[ipix,jpix,:] = True frames.setMask('BADPIXELS',badPix) # This checks for ADC & CL saturation print "photFlagSaturation", Date() frames = photFlagSaturation(frames, calTree=calTree, hkdata=hk, check='full') print "photConvDigit2Volts", Date() frames = photConvDigit2Volts(frames, calTree=calTree) print "Convert chopper position to angle", Date() frames = convertChopper2Angle(frames,calTree=calTree) System.gc() print "Add pointing", Date() frames = photAddInstantPointing(frames, pointing, calTree=calTree, orbitEphem = orbitEphem) System.gc() # we shall never use the pointing and ephemris data again print "cleanPlateauFrames ; " , Date() frames = cleanPlateauFrames(frames, calTree=calTree) System.gc() # now save the file print 'Write prepared frames to disk', Date() prepared_filename = os.path.join(self.path, os.path.basename(self.filename)) fa.save(prepared_filename + '_' + band + '_PreparedFrames.fits', frames) print
def runGC(): System.gc() time.sleep(1)
def L20_filterScanSpeed_kp(obs, camera): poolname = obs.level0.getCamera(camera).averaged.product.refs[0].urn.split( ':')[1] # ------------------------------------------------------------------------------ # Extract the calibration tree calTree = obs.calibration # # interactive user: you may apply following e.g. to get the most recent # calibration #calTree = getCalTree(obs=obs) # # ------------------------------------------------------------------------------ # # # Extract out the level1 from the ObservationContext level1 = PacsContext(obs.level1) frames = level1.averaged.getCamera(camera).product.getScience(0) # if not frames.containsMask('badprobs'): # print 'Run addgyroprobmask before running this script' # else: # ****************************************************************************** # Processing # ****************************************************************************** # Flag jump/module drop outs # frames = scanamorphosMaskLongTermGlitches(frames, stepAfter=20) # # Flag calibration block decays # frames = scanamorphosBaselinePreprocessing(frames) # # Filter on scan speed frames = runfilteronscanspeed(frames) if camera == "blue": # if PhotHelper.isParallelObs(obs): # pixsize = 3.2 # else: pixsize = 2. highpassradius = 15 else: pixsize = 3.2 highpassradius = 25 frames_hpf = highpassFilter(frames, highpassradius, copy=True, interpolateMaskedValues=True) System.gc() # # spatial deglitching now after just before photProject SPRs PACS-3522 & # PACS-3906 #s = Sigclip(10, 20) #s.behavior = Sigclip.CLIP #s.outliers = Sigclip.BOTH_OUTLIERS #s.mode = Sigclip.MEDIAN # #mdt = MapDeglitchTask() #mdt(frames, algo = s, deglitchvector="timeordered", calTree=calTree) # # save the new glitch mask to be casted back to L1 frames #if frames.meta.containsKey('Glitchmask') : # glitchmaskdata = frames.meta["Glitchmask"] # mask = frames.getMask(glitchmaskdata.value) # # Masking the "no scan data" (using BBID 215131301) - Mask name : NoScanData #frames = photMaskFrames(frames, noScanData = 1, remove=1) #PACS-4008 # #frames = filterOnScanSpeed(frames,lowScanSpeed=lowscanspeed,highScanSpeed=highscanspeed) # #System.gc() # prior source densities are insufficient to correct on a scan leg basis. # revisit the following lines if the situation changes. # scanindices = frames['Status']['ScanLineNumber'].data # numscanlegs = max(scanindices) # numtimesteps = len(scanindices) c1 = LocalStoreContext() c1_pre = c1.getStoreDir().toString().split('.hcss/')[0] dir_pre = c1_pre + 'correctscans/' if not os.path.exists(dir_pre): print 'Creating directory:' print dir_pre print 'for temp storage of scan leg maps' os.mkdir(dir_pre) # for i in xrange(20, 22): # selectindices = scanindices.where(scanindices == i) # masked pixels have a mask value of True # onlyselectindices = Bool3d(32, 64, numtimesteps, True) # onlyselectindices[:,:,selectindices] = False # frames.removeMask('onescan') # frames.addMaskType('onescan', 'only the ith scan') # frames.setMask('onescan', onlyselectindices) map, mi = photProject(frames_hpf, calTree=calTree, outputPixelsize=pixsize) coverage = map['coverage'].data cov_ind = coverage.where(coverage == 0.) map['image'].data[cov_ind] = Double.NaN map = centerRaDecMetaData(map) strobsid = str(obs.obsid) simpleFitsWriter(product=map['image'], file=dir_pre + 'scan' + strobsid + camera + '.fits') # # Add some Quality information to the frames frames = addQualityInformation(frames) # # Post processing # cast the glitch mask back into level 1 #if frames.meta.containsKey('Glitchmask') : # del(frames) # frames = level1.averaged.getCamera(camera).product.getScience(0) # frames.removeMask(glitchmaskdata.value) # frames.addMaskType(glitchmaskdata.value,glitchmaskdata.description) # frames.setMask(glitchmaskdata.value,mask) # frames.meta["Glitchmask"] = glitchmaskdata # level1.averaged.getCamera(camera).product.replace(0, frames) # obs.level1 = level1 # del(glitchmaskdata, mask) # # delete some variables #del level1, frames, hpfradius1, hpfradius2, outputPixelSize #del lowscanspeed, highscanspeed, speed, ref, med, index #del signal_stdev, cutlevel, threshold, HPFmask, s, mdt, image, ad, mi, historyCopy level1.averaged.getCamera(camera).product.replace(0, frames) obs.level1 = level1 saveObservation(obs, poolName=poolname, saveCalTree=True) return
# processing out-focus if unsynchronized: before = preprocess(target_dir + '\\out-focus\\before', out='out', filename='*') after = preprocess(target_dir + '\\out-focus\\after', out='out', filename='*') concatenate_files(before, after, target_dir + '\\out-focus\\merged\\merged.tif') else: preprocess(target_dir + '\\out-focus', out='out', filename='*').close() # processing caspase if unsynchronized: path_signal = target_dir + "\\caspase" path_imp = "\\caspasexy%sc1.tif" % pos path_imp_out = "\\caspasexy%sc1_sub.tif" % pos else: path_signal = target_dir + "\\caspase" path_imp = "\\seq0001xy%sc1.tif" % pos path_imp_out = "\\seq0001xy%sc1_sub.tif" % pos process_caspase_signal(path_signal, path_imp, path_imp_out) # processing pi signal process_pi_signal(target_dir, pos, unsynchronized=unsynchronized) System.gc()
def VesselFinder(channel_array, classifier_path): channels = channel_array image = channels[3] channels = channels[0:3] proc = image.getProcessor() directional_op = ImagePlus("directional_op", proc) tubes = range(5, 130, 12) img_source = ImagePlus("image", proc) src = clij2.push(img_source) dst = clij2.create(src) sigma = 2 clij2.gaussianBlur2D(src, dst, sigma, sigma) img_blur2 = clij2.pull(dst) src.close() dst.close() print("Tubeness mt start") tubenesses = [None] * len(tubes) rang = range(len(tubes)) threads = [] for i in rang: threads.append( threading.Thread(target=run_tube, args=(img_blur2, tubes[i], i, tubenesses))) threads[i].start() [x.join() for x in threads] print("Tubeness all done") print(tubenesses) src = clij2.push(img_source) dst = clij2.create(src) sigmas = [5, 20] imgsigmas = [] for sigma in sigmas: clij2.gaussianBlur2D(src, dst, sigma, sigma) img = clij2.pull(dst) imgsigmas.append(img) print("Gaussian Blur done") src.close() dst.close() variances = [5, 20] imgvars = [] for variance in variances: img = ImagePlus("image", proc) IJ.run(img, "Variance...", "radius=" + str(variance)) imgvars.append(img) print("Gaussian Blur done") featuresArray = FeatureStackArray(image.getStackSize()) stack = ImageStack(image.getWidth(), image.getHeight()) # add new feature here (2/2) and do not forget to add it with a # unique slice label! stack.addSlice("directional_op", directional_op.getProcessor()) for i in range(len(sigmas)): stack.addSlice("sigma" + str(sigmas[i]), imgsigmas[i].getProcessor()) for i in range(len(tubes)): stack.addSlice("Tubeness" + str(tubes[i]), tubenesses[i].getProcessor()) for i in range(len(variances)): stack.addSlice("Variance" + str(variances[i]), imgvars[i].getProcessor()) for i in range(len(channels)): stack.addSlice("channel" + str(i + 1), channels[i].getProcessor()) del sigmas del tubes del variances del channels # create empty feature stack features = FeatureStack(stack.getWidth(), stack.getHeight(), False) # set my features to the feature stack features.setStack(stack) # put my feature stack into the array featuresArray.set(features, 0) featuresArray.setEnabledFeatures(features.getEnabledFeatures()) del stack wekaSegmentation = WekaSegmentation(image) wekaSegmentation.setFeatureStackArray(featuresArray) wekaSegmentation.loadClassifier(classifier_path + "\\vessel-classifier_big.model") output = wekaSegmentation.applyClassifier(image, featuresArray, 0, True) System.gc() return output
def reset(self, randomize=False): for n in self.nodes: n.reset(randomize) # Force java garbage collection to free (hopefully) unused memory System.gc()
def windowClosed(self, event): System.gc()
totalee = 0 totalhe = 0 for i in range(nECB): parCl = colECB.getElementAt(i); fileOutECaloR.write( '{0:3} {1:4.4f} {2:4.4f} {3:4.4f} {4:4.4f} \n'.format(0,parCl.getPosition()[0],parCl.getPosition()[1],parCl.getPosition()[2],parCl.getEnergy()) ) # totalee += parCl.getEnergy() for i in range(nHCB): parCl = colHCB.getElementAt(i); fileOutHCaloR.write( '{0:3} {1:4.4f} {2:4.4f} {3:4.4f} {4:4.4f} \n'.format(1,parCl.getPosition()[0],parCl.getPosition()[1],parCl.getPosition()[2],parCl.getEnergy()) ) # totalhe += parCl.getEnergy() # print totalee, totalhe, totalee+totalhe # if nEvent > 10: break; del col del colPF del colCl del colTr del colECB del colHCB del evt reader.close() # close the file del reader System.gc()
def run1(): JyNI.JyRefMonitor_setMemDebugFlags(1) JyWeakReferenceGC.monitorNativeCollection = True # PyType <- Make native ref-cycle test with heap type to test partly-stub-mode. # PySet, but no native link to other PyObject # PyFrozenSet, " # PyCode, not GC-relevant in CPython import DemoExtension #Note: # For now we attempt to verify JyNI's GC-functionality independently from # Jython concepts like Jython weak references or Jython GC-module. # So we use java.lang.ref.WeakReference and java.lang.System.gc to monitor # and control Java-gc. #JyNI.JyRefMonitor_setMemDebugFlags(1) #JyWeakReferenceGC.monitorNativeCollection = True #l = (123,) l = ([0, "test"],) l[0][0] = l #l = (123, {'a': 0, 'b': "test"}) #l[1]['a'] = l #We create weak reference to l to monitor collection by Java-GC: wkl = WeakReference(l) print "weak(l): "+str(wkl.get()) # We pass down l to some native method. We don't care for the method itself, # but conversion to native side causes creation of native PyObjects that # correspond to l and its elements. We will then track the life-cycle of these. print "make l native..." DemoExtension.argCountToString(l) #print "argCountToString.__doc__-address: "+str(JyNI.lookupNativeHandle(DemoExtension.argCountToString.__doc__)) #print "We access a method-doc as this used to cause problems with GC:" #print " "+DemoExtension.argCountToString.__doc__ #print "Native static objects so far:" #print JyNI.nativeStaticPyObjectHeads.keySet() print "Delete l... (but GC not yet ran)" del l #l = None print "weak(l) after del: "+str(wkl.get()) print "" # monitor.list-methods display the following format: # [native pointer]{'' | '_GC_J' | '_J'} ([type]) #[native ref-count]: [repr] *[creation time] # _GC_J means that JyNI tracks the object # _J means that a JyNI-GC-head exists, but the object is not actually treated by GC # This can serve monitoring purposes or soft-keep-alive (c.f. java.lang.ref.SoftReference) # for caching. print "Leaks before GC:" monitor.listLeaks() print "" # By inserting this line you can confirm that native # leaks would persist if JyNI-GC is not working: #JyWeakReferenceGC.nativecollectionEnabled = False print "calling Java-GC..." System.gc() time.sleep(2) print "weak(l) after GC: "+str(wkl.get()) print "" monitor.listWouldDeleteNative() print "" print "leaks after GC:" monitor.listLeaks() print "" print " "+DemoExtension.argCountToString.__doc__ monitor.listLeaks() #print "----" #print monitor.listAll() print "" print "====" print "exit" print "===="
def reset(self,randomize=False): for n in self.nodes: n.reset(randomize) # Force java garbage collection to free (hopefully) unused memory System.gc()
def run3(): JyNI.JyRefMonitor_setMemDebugFlags(1) JyWeakReferenceGC.monitorNativeCollection = True import DemoExtension #JyNI.JyRefMonitor_setMemDebugFlags(1) #JyWeakReferenceGC.monitorNativeCollection = True l = [0, "test1"] # print l # DemoExtension.listSetIndex(l, 0, 100.7) # print l d = {'a': 7, 'b': "test6"} #We create weak reference to l to monitor collection by Java-GC: wkl = WeakReference(l) wkd = WeakReference(d) wkl2 = weakref.ref(l) wkd2 = weakref.ref(d) #Note: #===== #To make this work we'll have to ensure that d gets a GCHead even though #l doesn't recognize the insertion of d and thus would not explore it. #In fact every C-stub object must get a GCHead when going native, regardless #of whether it is inserted somewhere or not, because it might be silently #inserted and a GCHead is the only way to detect this and fix it. #Add a call (e.g. to size) to the dict in l on native side to test that #d still works (which it currently shouldn't). #Later use d's GCHead to detect the invalid graph and resurrect d's java-part. #Take care of the GIL here. CStubs must release the GIL when they detect a #failing backend, so the GC-thread can acquire it and resurrect the backend. #How to fix the weak reference etc? #l[0] = d print "weak(l): "+str(wkl.get()) print "weak(d): "+str(wkd.get()) print "weak2(l): "+str(wkl2()) print "weak2(d): "+str(wkd2()) print "make l native..." #l[0] = d #DemoExtension.argCountToString(l) #l[0] = d DemoExtension.listSetIndex(l, 0, d) print "Delete l... (but GC not yet ran)" #del l del d #l = None #print "weak(l) after del: "+str(wkl.get()) print "weak(d) after del: "+str(wkd.get()) print "weak2(d) after del: "+str(wkd2()) print "" print "Leaks before GC:" monitor.listLeaks() print "" print "calling Java-GC..." System.gc() time.sleep(2) # if monitor.lastClearGraphValid: # print "valid graph" # else: # print "invalid graph" # monitor.lastClearGraphValid = False print "weak(l) after GC: "+str(wkl.get()) #print "l after GC: "+str(l) print "weak(d) after GC: "+str(wkd.get()) print "weak2(l) after GC: "+str(wkl2()) print "weak2(d) after GC: "+str(wkd2()) wkd = WeakReference(l[0]) print "" #monitor.listWouldDeleteNative() print "" #print "leaks after GC:" #monitor.listLeaks() print "l[0], i.e. d after gc:" #print l[0] #print len(l[0]) print "------" print "del l..." del l System.gc() time.sleep(2) # if monitor.lastClearGraphValid: # print "valid graph" # else: # print "invalid graph" # monitor.lastClearGraphValid = False print "" monitor.listWouldDeleteNative() print "" #print "leaks after GC (l deleted):" #monitor.listLeaks() #print DemoExtension.argCountToString.__doc__ #monitor.listFreeStatus("dict") #monitor.listAll() #GlobalRef.processDelayedCallbacks() print "weak(d) after GC2: "+str(wkd.get()) print "weak2(l) after GC2: "+str(wkl2()) print "weak2(d) after GC2: "+str(wkd2()) System.gc() time.sleep(2) print "weak(d) after GC3: "+str(wkd.get()) monitor.listWouldDeleteNative() print "" print "leaks after GC3:" monitor.listLeaks() print "" print "====" print "exit" print "===="
def run1(): JyNI.JyRefMonitor_setMemDebugFlags(1) JyWeakReferenceGC.monitorNativeCollection = True # PyType <- Make native ref-cycle test with heap type to test partly-stub-mode. # PySet, but no native link to other PyObject # PyFrozenSet, " # PyCode, not GC-relevant in CPython import DemoExtension #Note: # For now we attempt to verify JyNI's GC-functionality independently from # Jython concepts like Jython weak references or Jython GC-module. # So we use java.lang.ref.WeakReference and java.lang.System.gc to monitor # and control Java-gc. #JyNI.JyRefMonitor_setMemDebugFlags(1) #JyWeakReferenceGC.monitorNativeCollection = True #l = (123,) l = ([0, "test"], ) l[0][0] = l #l = (123, {'a': 0, 'b': "test"}) #l[1]['a'] = l #We create weak reference to l to monitor collection by Java-GC: wkl = WeakReference(l) print "weak(l): " + str(wkl.get()) # We pass down l to some native method. We don't care for the method itself, # but conversion to native side causes creation of native PyObjects that # correspond to l and its elements. We will then track the life-cycle of these. print "make l native..." DemoExtension.argCountToString(l) #print "argCountToString.__doc__-address: "+str(JyNI.lookupNativeHandle(DemoExtension.argCountToString.__doc__)) #print "We access a method-doc as this used to cause problems with GC:" #print " "+DemoExtension.argCountToString.__doc__ #print "Native static objects so far:" #print JyNI.nativeStaticPyObjectHeads.keySet() print "Delete l... (but GC not yet ran)" del l #l = None print "weak(l) after del: " + str(wkl.get()) print "" # monitor.list-methods display the following format: # [native pointer]{'' | '_GC_J' | '_J'} ([type]) #[native ref-count]: [repr] *[creation time] # _GC_J means that JyNI tracks the object # _J means that a JyNI-GC-head exists, but the object is not actually treated by GC # This can serve monitoring purposes or soft-keep-alive (c.f. java.lang.ref.SoftReference) # for caching. print "Leaks before GC:" monitor.listLeaks() print "" # By inserting this line you can confirm that native # leaks would persist if JyNI-GC is not working: #JyWeakReferenceGC.nativecollectionEnabled = False print "calling Java-GC..." System.gc() time.sleep(2) print "weak(l) after GC: " + str(wkl.get()) print "" monitor.listWouldDeleteNative() print "" print "leaks after GC:" monitor.listLeaks() print "" print " " + DemoExtension.argCountToString.__doc__ monitor.listLeaks() #print "----" #print monitor.listAll() print "" print "====" print "exit" print "===="
def run3(): JyNI.JyRefMonitor_setMemDebugFlags(1) JyWeakReferenceGC.monitorNativeCollection = True import DemoExtension #JyNI.JyRefMonitor_setMemDebugFlags(1) #JyWeakReferenceGC.monitorNativeCollection = True l = [0, "test1"] # print l # DemoExtension.listSetIndex(l, 0, 100.7) # print l d = {'a': 7, 'b': "test6"} #We create weak reference to l to monitor collection by Java-GC: wkl = WeakReference(l) wkd = WeakReference(d) wkl2 = weakref.ref(l) wkd2 = weakref.ref(d) #Note: #===== #To make this work we'll have to ensure that d gets a GCHead even though #l doesn't recognize the insertion of d and thus would not explore it. #In fact every C-stub object must get a GCHead when going native, regardless #of whether it is inserted somewhere or not, because it might be silently #inserted and a GCHead is the only way to detect this and fix it. #Add a call (e.g. to size) to the dict in l on native side to test that #d still works (which it currently shouldn't). #Later use d's GCHead to detect the invalid graph and resurrect d's java-part. #Take care of the GIL here. CStubs must release the GIL when they detect a #failing backend, so the GC-thread can acquire it and resurrect the backend. #How to fix the weak reference etc? #l[0] = d print "weak(l): " + str(wkl.get()) print "weak(d): " + str(wkd.get()) print "weak2(l): " + str(wkl2()) print "weak2(d): " + str(wkd2()) print "make l native..." #l[0] = d #DemoExtension.argCountToString(l) #l[0] = d DemoExtension.listSetIndex(l, 0, d) print "Delete l... (but GC not yet ran)" #del l del d #l = None #print "weak(l) after del: "+str(wkl.get()) print "weak(d) after del: " + str(wkd.get()) print "weak2(d) after del: " + str(wkd2()) print "" print "Leaks before GC:" monitor.listLeaks() print "" print "calling Java-GC..." System.gc() time.sleep(2) # if monitor.lastClearGraphValid: # print "valid graph" # else: # print "invalid graph" # monitor.lastClearGraphValid = False print "weak(l) after GC: " + str(wkl.get()) #print "l after GC: "+str(l) print "weak(d) after GC: " + str(wkd.get()) print "weak2(l) after GC: " + str(wkl2()) print "weak2(d) after GC: " + str(wkd2()) wkd = WeakReference(l[0]) print "" #monitor.listWouldDeleteNative() print "" #print "leaks after GC:" #monitor.listLeaks() print "l[0], i.e. d after gc:" #print l[0] #print len(l[0]) print "------" print "del l..." del l System.gc() time.sleep(2) # if monitor.lastClearGraphValid: # print "valid graph" # else: # print "invalid graph" # monitor.lastClearGraphValid = False print "" monitor.listWouldDeleteNative() print "" #print "leaks after GC (l deleted):" #monitor.listLeaks() #print DemoExtension.argCountToString.__doc__ #monitor.listFreeStatus("dict") #monitor.listAll() #GlobalRef.processDelayedCallbacks() print "weak(d) after GC2: " + str(wkd.get()) print "weak2(l) after GC2: " + str(wkl2()) print "weak2(d) after GC2: " + str(wkd2()) System.gc() time.sleep(2) print "weak(d) after GC3: " + str(wkd.get()) monitor.listWouldDeleteNative() print "" print "leaks after GC3:" monitor.listLeaks() print "" print "====" print "exit" print "===="