def test_raw_forced_delayed(self):
comments = []
class Test_JavaAbortFinalizable(Object):
def __init__(self, name, toAbort):
self.name = name
self.toAbort = toAbort
def __repr__(self):
return "<" + self.name + ">"
def finalize(self):
gc.notifyPreFinalization()
comments.append("del " + self.name)
gc.abortDelayedFinalization(self.toAbort)
# We manually restore weak references:
gc.restoreWeakReferences(self.toAbort)
gc.notifyPostFinalization()
class Test_Finalizable(object):
def __init__(self, name):
self.name = name
def __repr__(self):
return "<" + self.name + ">"
def __del__(self):
comments.append("del " + self.name)
def callback_a(obj):
comments.append("callback_a")
def callback_b(obj):
comments.append("callback_b")
a = Test_Finalizable("a")
wa = weakref.ref(a, callback_a)
b = Test_JavaAbortFinalizable("b", a)
wb = weakref.ref(b, callback_b)
gc.addJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION)
gc.addJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS)
self.assertTrue(gc.delayedFinalizationEnabled())
self.assertTrue(gc.delayedWeakrefCallbacksEnabled())
self.assertEqual(len(comments), 0)
del a
del b
System.gc()
time.sleep(2)
self.assertIsNotNone(wa())
self.assertIsNone(wb())
self.assertIn('del b', comments)
self.assertNotIn('callback_a', comments)
self.assertIn('callback_b', comments)
self.assertNotIn('del a', comments)
self.assertEqual(2, len(comments))
gc.removeJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION)
gc.removeJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS)
def test_raw_forced_delayed(self):
comments = []
class Test_JavaAbortFinalizable(Object):
def __init__(self, name, toAbort):
self.name = name
self.toAbort = toAbort
def __repr__(self):
return "<"+self.name+">"
def finalize(self):
gc.notifyPreFinalization()
comments.append("del "+self.name)
gc.abortDelayedFinalization(self.toAbort)
# We manually restore weak references:
gc.restoreWeakReferences(self.toAbort)
gc.notifyPostFinalization()
class Test_Finalizable(object):
def __init__(self, name):
self.name = name
def __repr__(self):
return "<"+self.name+">"
def __del__(self):
comments.append("del "+self.name)
def callback_a(obj):
comments.append("callback_a")
def callback_b(obj):
comments.append("callback_b")
a = Test_Finalizable("a")
wa = weakref.ref(a, callback_a)
b = Test_JavaAbortFinalizable("b", a)
wb = weakref.ref(b, callback_b)
gc.addJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION)
gc.addJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS)
self.assertTrue(gc.delayedFinalizationEnabled())
self.assertTrue(gc.delayedWeakrefCallbacksEnabled())
self.assertEqual(len(comments), 0)
del a
del b
System.gc()
time.sleep(2)
self.assertIsNotNone(wa())
self.assertIsNone(wb())
self.assertIn('del b', comments)
self.assertNotIn('callback_a', comments)
self.assertIn('callback_b', comments)
self.assertNotIn('del a', comments)
self.assertEqual(2, len(comments))
gc.removeJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION)
gc.removeJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS)
def test_raw_forced_delayedWeakrefCallback(self):
comments = []
resurrected = []
class Test_JavaResurrectFinalizable(Object):
def __init__(self, name, toResurrect):
self.name = name
self.toResurrect = toResurrect
def __repr__(self):
return "<"+self.name+">"
# Note that this type of finalizer is usually not recommended
# as it gets lost in case of resurrection.
def finalize(self):
gc.notifyPreFinalization()
comments.append("del "+self.name)
resurrected.append(self.toResurrect)
# We manually restore weak references:
gc.restoreWeakReferences(self.toResurrect)
gc.notifyPostFinalization()
class Test_Finalizable(object):
def __init__(self, name):
self.name = name
def __repr__(self):
return "<"+self.name+">"
def __del__(self):
comments.append("del "+self.name)
def callback(obj):
comments.append("callback")
a = Test_Finalizable("a")
b = Test_JavaResurrectFinalizable("b", a)
wa = weakref.ref(a, callback)
gc.removeJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION)
gc.addJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS)
self.assertFalse(gc.delayedFinalizationEnabled())
self.assertTrue(gc.delayedWeakrefCallbacksEnabled())
self.assertEqual(len(comments), 0)
aStr = str(a)
del a
del b
System.gc()
time.sleep(2)
self.assertIn("del a", comments)
self.assertIn("del b", comments)
self.assertEqual(1, len(resurrected))
self.assertEqual(str(resurrected[0]), aStr)
self.assertIsNotNone(wa())
self.assertEqual(resurrected[0], wa())
self.assertNotIn("callback", comments)
self.assertEqual(2, len(comments))
gc.removeJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS)
def test_raw_forced_delayedWeakrefCallback(self):
comments = []
resurrected = []
class Test_JavaResurrectFinalizable(Object):
def __init__(self, name, toResurrect):
self.name = name
self.toResurrect = toResurrect
def __repr__(self):
return "<" + self.name + ">"
# Note that this type of finalizer is usually not recommended
# as it gets lost in case of resurrection.
def finalize(self):
gc.notifyPreFinalization()
comments.append("del " + self.name)
resurrected.append(self.toResurrect)
# We manually restore weak references:
gc.restoreWeakReferences(self.toResurrect)
gc.notifyPostFinalization()
class Test_Finalizable(object):
def __init__(self, name):
self.name = name
def __repr__(self):
return "<" + self.name + ">"
def __del__(self):
comments.append("del " + self.name)
def callback(obj):
comments.append("callback")
a = Test_Finalizable("a")
b = Test_JavaResurrectFinalizable("b", a)
wa = weakref.ref(a, callback)
gc.removeJythonGCFlags(gc.FORCE_DELAYED_FINALIZATION)
gc.addJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS)
self.assertFalse(gc.delayedFinalizationEnabled())
self.assertTrue(gc.delayedWeakrefCallbacksEnabled())
self.assertEqual(len(comments), 0)
aStr = str(a)
del a
del b
System.gc()
time.sleep(2)
self.assertIn("del a", comments)
self.assertIn("del b", comments)
self.assertEqual(1, len(resurrected))
self.assertEqual(str(resurrected[0]), aStr)
self.assertIsNotNone(wa())
self.assertEqual(resurrected[0], wa())
self.assertNotIn("callback", comments)
self.assertEqual(2, len(comments))
gc.removeJythonGCFlags(gc.FORCE_DELAYED_WEAKREF_CALLBACKS)
def runGCIfJython():
try:
currentIndex = GCDetector.gcIndex
gcCount = 0
detector = GCDetector()
detector = None
while currentIndex == GCDetector.gcIndex and gcCount < maxGCRun:
System.gc()
gcCount += 1
time.sleep(0.1)
except:
pass
def test_finalization_preprocess_and_postprocess(self):
# Note that this test is done here again (already was in another class
# in this module), to see that everything works as it should also with
# a different flag-context.
comments = []
self0 = self
class A:
def __del__(self):
self0.assertIn("run PreProcess", comments)
comments.append("A del")
# let's simulate a time-consuming finalizer
# to ensure that post finalization processing
# is sensitive to this
time.sleep(0.5)
comments.append("A del done")
class PreProcess(Runnable):
def run(self):
self0.assertEqual(comments, [])
comments.append("run PreProcess")
class PostProcess(Runnable):
def run(self):
self0.assertIn("run PreProcess", comments)
self0.assertIn("A del", comments)
self0.assertIn("A del done", comments)
comments.append("run PostProcess")
a = A()
a = None
prePr = PreProcess()
postPr = PostProcess()
time.sleep(1) # <- to avoid that the newly registered processes
# become subject to previous run (remember: We
# are not in monitor-mode, i.e. gc runs async.
gc.registerPreFinalizationProcess(prePr)
gc.registerPostFinalizationProcess(postPr)
# Note that order matters here:
# If the flag gc.DONT_FINALIZE_RESURRECTED_OBJECTS is used,
# gc.registerPostFinalizationProcess(postPr, 0) would lead to failure,
# because postPr asserts that a's finalizer already ran. Since
# DONT_FINALIZE_RESURRECTED_OBJECTS also inserted a postprocess,
# to perform delayed finalization, the 0-index would prepend postPr
# before the process that actually runs the finalizers.
System.gc()
# we wait a bit longer here, since PostProcess runs asynchronous
# and must wait for the finalizer of A
time.sleep(2)
self.assertIn("run PostProcess", comments)
comments = []
gc.unregisterPreFinalizationProcess(prePr)
gc.unregisterPostFinalizationProcess(postPr)
def test_finalization_preprocess_and_postprocess(self):
#Note that this test is done here again (already was in another class
#in this module), to see that everything works as it should also with
#a different flag-context.
comments = []
self0 = self
class A:
def __del__(self):
self0.assertIn("run PreProcess", comments)
comments.append("A del")
#let's simulate a time-consuming finalizer
#to ensure that post finalization processing
#is sensitive to this
time.sleep(0.5)
comments.append("A del done")
class PreProcess(Runnable):
def run(self):
self0.assertEqual(comments, [])
comments.append("run PreProcess")
class PostProcess(Runnable):
def run(self):
self0.assertIn("run PreProcess", comments)
self0.assertIn("A del", comments)
self0.assertIn("A del done", comments)
comments.append("run PostProcess")
a = A()
a = None
prePr = PreProcess()
postPr = PostProcess()
time.sleep(1) # <- to avoid that the newly registered processes
# become subject to previous run (remember: We
# are not in monitor-mode, i.e. gc runs async.
gc.registerPreFinalizationProcess(prePr)
gc.registerPostFinalizationProcess(postPr)
#Note that order matters here:
#If the flag gc.DONT_FINALIZE_RESURRECTED_OBJECTS is used,
#gc.registerPostFinalizationProcess(postPr, 0) would lead to failure,
#because postPr asserts that a's finalizer already ran. Since
#DONT_FINALIZE_RESURRECTED_OBJECTS also inserted a postprocess,
#to perform delayed finalization, the 0-index would prepend postPr
#before the process that actually runs the finalizers.
System.gc()
#we wait a bit longer here, since PostProcess runs asynchronous
#and must wait for the finalizer of A
time.sleep(2)
self.assertIn("run PostProcess", comments)
comments = []
gc.unregisterPreFinalizationProcess(prePr)
gc.unregisterPostFinalizationProcess(postPr)
def assertTriggered(test, expected):
deadline = time() + DelTrigger.TIMEOUT
while time() < deadline:
System.gc()
System.runFinalization()
try:
test.assertEqual(expected, DelTrigger.triggered)
return
except AssertionError:
if not expected:
raise
sleep(DelTrigger.TIMEOUT / 10)
test.fail("Not triggered after {} seconds".format(DelTrigger.TIMEOUT))
def test_weakref_after_resurrection_threadsafe(self):
resurrect = []
comments = []
class Test_Finalizable(object):
def __init__(self, name):
self.name = name
def __repr__(self):
return "<"+self.name+">"
def __del__(self):
comments.append("del "+self.name)
class Test_Resurrection(object):
def __init__(self, name):
self.name = name
def __repr__(self):
return "<"+self.name+">"
def __del__(self):
comments.append("del "+self.name)
if hasattr(self, "toResurrect"):
resurrect.append(self)
a = Test_Finalizable("a")
wa = weakref.ref(a)
c = Test_Resurrection("c")
c.toResurrect = a
wc = weakref.ref(c)
del a
del c
try:
gc.addJythonGCFlags(gc.PRESERVE_WEAKREFS_ON_RESURRECTION)
System.gc()
# We intentionally don't wait here, but want to observe
# the situation with gc unfinnished. Note that wa() preserves
# its result right away, due to thread-safe implementation.
# Technically, the weak reference breaks and is restored after
# gc-run finishes. However wa() blocks until the referent is
# restored or the deletion is confirmed.
except Exception:
pass
self.assertEqual(comments, [])
self.assertEqual(resurrect, [])
while comments == [] or resurrect == []:
self.assertEqual(str(wa()), '<a>')
self.assertEqual(wc(), None)
self.assertEqual(str(wa()), '<a>')
self.assertEqual(wc(), None)
def test_weakref_after_resurrection_threadsafe(self):
resurrect = []
comments = []
class Test_Finalizable(object):
def __init__(self, name):
self.name = name
def __repr__(self):
return "<" + self.name + ">"
def __del__(self):
comments.append("del " + self.name)
class Test_Resurrection(object):
def __init__(self, name):
self.name = name
def __repr__(self):
return "<" + self.name + ">"
def __del__(self):
comments.append("del " + self.name)
if hasattr(self, "toResurrect"):
resurrect.append(self)
a = Test_Finalizable("a")
wa = weakref.ref(a)
c = Test_Resurrection("c")
c.toResurrect = a
wc = weakref.ref(c)
del a
del c
try:
gc.addJythonGCFlags(gc.PRESERVE_WEAKREFS_ON_RESURRECTION)
System.gc()
# We intentionally don't wait here, but want to observe
# the situation with gc unfinnished. Note that wa() preserves
# its result right away, due to thread-safe implementation.
# Technically, the weak reference breaks and is restored after
# gc-run finishes. However wa() blocks until the referent is
# restored or the deletion is confirmed.
except Exception:
pass
self.assertEqual(comments, [])
self.assertEqual(resurrect, [])
while comments == [] or resurrect == []:
self.assertEqual(str(wa()), '<a>')
self.assertEqual(wc(), None)
self.assertEqual(str(wa()), '<a>')
self.assertEqual(wc(), None)
def populateMusicList(self):
#If there are files
if not MusicPlayer.isFolderEmpty():
walker = Files.walk(Paths.get('../resources/music'))
#Ridiculous that we should have to do this. Nevertheless, here goes.
class musicFilterPred(Predicate):
#@Override
def test(self, song):
#No Symbolic Links
if Files.isRegularFile(song, LinkOption.NOFOLLOW_LINKS):
# MP3, WAV, AAC only, for now.
file = String(song.toFile().toURI().toString())
ext = file.substring(file.length() - 4,
file.length()) #.XYZ
if ext not in ('.wav', '.WAV', '.mp3', '.MP3', '.aac',
'.AAC'):
return False
else:
#We are presuming that a file with the correct extension is actually
#of the format specified by the extension and a valid file.
#We will handle if is not, but for now, filter out the obvious bad eggs.
return True
from java.util.function import Function
class musicMapPred(Function):
#@Override
def apply(self, song):
return Media(song.toFile().toURI().toString())
#The Result
music_list = walker.filter(musicFilterPred()).map(
musicMapPred()).collect(Collectors.toList())
#Close Stream
walker.close()
walker = None
#Nudge GC
System.gc()
#Retun valid Music Lists only
if music_list.isEmpty():
return None
else:
return music_list
#No files
else:
return None
def get_jython_free_mem(self):
# Force garbage collection first.
from java.lang import Runtime
from java.lang.ref import WeakReference
from java.lang import Object
from java.lang import System
obj = Object()
ref = WeakReference(obj)
obj = None
while ref.get() is not None:
System.gc()
# Calculate approx. available memory.
return Runtime.getRuntime().freeMemory()
def run4():
import DemoExtension
JyNI.JyRefMonitor_setMemDebugFlags(1)
l = ([0, "test"], )
l[0][0] = l
DemoExtension.argCountToString(l)
del l
#l[0][1] = None
print "Leaks before GC:"
monitor.listLeaks()
System.gc()
time.sleep(2)
print "Leaks after GC:"
monitor.listLeaks()
def run4(): import DemoExtension JyNI.JyRefMonitor_setMemDebugFlags(1) l = ([0, "test"],) l[0][0] = l DemoExtension.argCountToString(l) del l #l[0][1] = None print "Leaks before GC:" monitor.listLeaks() System.gc() time.sleep(2) print "Leaks after GC:" monitor.listLeaks()
def test_forced_with_extern_NonPyObjectFinalizer_that_notifies_gc(self):
comments = []
class A:
def __init__(self, index):
self.index = index
def __del__(self):
comments.append("A_del_" + str(self.index))
class PreProcess(Runnable):
preCount = 0
def run(self):
PreProcess.preCount += 1
class PostProcess(Runnable):
postCount = 0
def run(self):
PostProcess.postCount += 1
prePr = PreProcess()
postPr = PostProcess()
time.sleep(1) # <- to avoid that the newly registered processes
# become subject to previous run (remember: We
# are not in monitor-mode, i.e. gc runs async.
gc.registerPreFinalizationProcess(prePr)
gc.registerPostFinalizationProcess(postPr)
for i in range(4):
f = A(i)
del f
#NastyFinalizer would cause this test occasionally to fail
externFinalizer = GCTestHelper.NotSoNastyFinalizer()
del externFinalizer
for i in range(4, 8):
f = A(i)
del f
System.gc()
# we wait a bit longer here, since PostProcess runs asynchronous
# and must wait for the finalizer of A
time.sleep(4)
self.assertEqual(len(comments), 8)
self.assertEqual(PreProcess.preCount, 1)
self.assertEqual(PostProcess.postCount, 1)
comments = []
gc.unregisterPreFinalizationProcess(prePr)
gc.unregisterPostFinalizationProcess(postPr)
def test_with_extern_NonPyObjectFinalizer_that_notifies_gc(self):
comments = []
class A:
def __init__(self, index):
self.index = index
def __del__(self):
comments.append("A_del_"+str(self.index))
class PreProcess(Runnable):
preCount = 0
def run(self):
PreProcess.preCount += 1
class PostProcess(Runnable):
postCount = 0
def run(self):
PostProcess.postCount += 1
prePr = PreProcess()
postPr = PostProcess()
time.sleep(1) # <- to avoid that the newly registered processes
# become subject to previous run (remember: We
# are not in monitor-mode, i.e. gc runs async.
gc.registerPreFinalizationProcess(prePr)
gc.registerPostFinalizationProcess(postPr)
for i in range(4):
f = A(i)
del f
#NastyFinalizer would cause this test occasionally to fail
externFinalizer = GCTestHelper.NotSoNastyFinalizer()
del externFinalizer
for i in range(4, 8):
f = A(i)
del f
System.gc()
#we wait a bit longer here, since PostProcess runs asynchronous
#and must wait for the finalizer of A
time.sleep(4)
self.assertEqual(len(comments), 8)
self.assertEqual(PreProcess.preCount, 1)
self.assertEqual(PostProcess.postCount, 1)
comments = []
gc.unregisterPreFinalizationProcess(prePr)
gc.unregisterPostFinalizationProcess(postPr)
def load_eol_ids(inpath, tax):
System.gc()
rt = Runtime.getRuntime()
print '# Memory', rt.freeMemory()/(1024*1024), rt.totalMemory()/(1024*1024)
page_to_nodes = {}
node_to_pages = {}
with open(inpath, 'r') as infile:
print '| Processing EOL page ids file %s' % (inpath)
reader = csv.reader(infile)
row_count = 0
hits = 0
for row in reader:
row_count += 1
[page_id, idspace, source_id] = row
if row_count % 250000 == 0:
print row_count, row
qid = QualifiedId(idspace, source_id)
node = tax.lookupQid(qid)
if node != None: # and node.isPotentialOtu():
nodes = page_to_nodes.get(page_id)
if nodes == None:
nodes = [(node, qid)]
page_to_nodes[page_id] = nodes
elif not in_alist(node, nodes):
nodes.append((node, qid))
pages = node_to_pages.get(node)
if pages == None:
pages = [page_id]
node_to_pages[node] = pages
elif not page_id in pages:
pages.append(page_id)
print '| OTT nodes having at least one EOL page: %s' % len(node_to_pages)
print '| EOL page ids having at least one OTT node: %s' % len(page_to_nodes)
# Sort page ids for each OTT node (will use smallest one)
for node in node_to_pages:
pages = node_to_pages[node]
if len(pages) > 1:
pages.sort(key=int)
return (page_to_nodes, node_to_pages)
def main():
from sys import argv
if len(argv) == 2:
classToFiles = lambda x: classToFile(x.__name__)
if argv[1] == 'all':
# traverse throught all listed packages
for package in [com, java, javax, org]: # sun
print 'proceed package', package.__name__
packageToFile(package)
classTraverse(package, classToFiles, packageToFile)
System.gc()
print 'writen ', len(classSaved), ' files'
while len(classSaved) != 0:
del classSaved[0]
elif argv[1] == 'packages':
for package in [com, java, javax, org]:
print 'proceed package', package.__name__
packageToFile(package)
classTraverse(package, lambda x: None, packageToFile)
System.gc()
elif argv[1] == 'css':
toFile('class.css', classCss(2))
elif argv[1] == 'index':
print 'creating index.html'
toFile('index.html', frameSet())
lnk = '<a href="%s.html">%s</a>'
packages = [com, java, javax, org]
links = [lnk % (p.__name__, p.__name__) for p in packages]
start = ['<h1>(JYTHON) CLASS</h1>'] + ['<ul>']
start += ul('ind0', links)
start += ['</ul>']
html = join(start, (' ' * 4) + '\n')
print 'creating start.html'
toFile('start.html', baseHtml('javaDoc', html))
else:
arg = eval(argv[1])
if type(arg) == PyJavaPackage:
classTraverse(arg, classToFileS, packageToFile)
elif type(arg) != PyJavaPackage:
classToFile(argv[1])
else:
usage()
else:
usage()
def execute(self):
import herschel, os
obsCont = fa.load(self.filename + "_ObsCont.fits")
hk = fa.load(self.filename + "_HPPHK.fits")["hk"]
pointing = fa.load(self.filename + "_Pointing.fits")
orbitEphem = fa.load(self.filename + "_OrbitEphem.fits")
timeCorr = fa.load(self.filename + "_TimeCorr.fits")
calTree = getCalTree("FM")
for band in ("blue", "green", "red"):
try:
frames = fa.load(self.filename + "_" + band + "_level0Frames.fits")
except java.io.IOException:
continue
print "running findBlocks", Date()
frames = findBlocks(frames)
print "photFlagBadPixels", Date()
frames = photFlagBadPixels(frames, calTree=calTree)
myMask = _mkMyMask(band)
badPix = frames.getMask("BADPIXELS")
dims = badPix.dimensions
for ipix in range(dims[0]):
for jpix in range(dims[1]):
if myMask[ipix, jpix] == True:
badPix[ipix, jpix, :] = True
frames.setMask("BADPIXELS", badPix)
# This checks for ADC & CL saturation
print "photFlagSaturation", Date()
frames = photFlagSaturation(frames, calTree=calTree, hkdata=hk, check="full")
print "photConvDigit2Volts", Date()
frames = photConvDigit2Volts(frames, calTree=calTree)
print "Convert chopper position to angle", Date()
frames = convertChopper2Angle(frames, calTree=calTree)
System.gc()
print "Add pointing", Date()
frames = photAddInstantPointing(frames, pointing, calTree=calTree, orbitEphem=orbitEphem)
System.gc()
# we shall never use the pointing and ephemris data again
print "cleanPlateauFrames ; ", Date()
frames = cleanPlateauFrames(frames, calTree=calTree)
System.gc()
# now save the file
print "Write prepared frames to disk", Date()
prepared_filename = os.path.join(self.path, os.path.basename(self.filename))
fa.save(prepared_filename + "_" + band + "_PreparedFrames.fits", frames)
print
def execute(self):
import herschel, os
obsCont = fa.load(self.filename + '_ObsCont.fits')
hk = fa.load(self.filename + '_HPPHK.fits')['hk']
pointing = fa.load(self.filename + '_Pointing.fits')
orbitEphem = fa.load(self.filename + '_OrbitEphem.fits')
timeCorr = fa.load(self.filename + '_TimeCorr.fits')
calTree = getCalTree("FM")
for band in ('blue', 'green', 'red'):
try:
frames = fa.load(self.filename + '_' + band + '_level0Frames.fits')
except java.io.IOException:
continue
print "running findBlocks", Date()
frames = findBlocks(frames)
print "photFlagBadPixels", Date()
frames = photFlagBadPixels(frames, calTree=calTree)
myMask = _mkMyMask(band)
badPix = frames.getMask('BADPIXELS')
dims = badPix.dimensions
for ipix in range(dims[0]):
for jpix in range(dims[1]):
if (myMask[ipix,jpix] == True):
badPix[ipix,jpix,:] = True
frames.setMask('BADPIXELS',badPix)
# This checks for ADC & CL saturation
print "photFlagSaturation", Date()
frames = photFlagSaturation(frames, calTree=calTree, hkdata=hk, check='full')
print "photConvDigit2Volts", Date()
frames = photConvDigit2Volts(frames, calTree=calTree)
print "Convert chopper position to angle", Date()
frames = convertChopper2Angle(frames,calTree=calTree)
System.gc()
print "Add pointing", Date()
frames = photAddInstantPointing(frames, pointing, calTree=calTree, orbitEphem = orbitEphem)
System.gc()
# we shall never use the pointing and ephemris data again
print "cleanPlateauFrames ; " , Date()
frames = cleanPlateauFrames(frames, calTree=calTree)
System.gc()
# now save the file
print 'Write prepared frames to disk', Date()
prepared_filename = os.path.join(self.path, os.path.basename(self.filename))
fa.save(prepared_filename + '_' + band + '_PreparedFrames.fits', frames)
print
def runGC(): System.gc() time.sleep(1)
def L20_filterScanSpeed_kp(obs, camera):
poolname = obs.level0.getCamera(camera).averaged.product.refs[0].urn.split(
':')[1]
# ------------------------------------------------------------------------------
# Extract the calibration tree
calTree = obs.calibration
#
# interactive user: you may apply following e.g. to get the most recent
# calibration
#calTree = getCalTree(obs=obs)
#
# ------------------------------------------------------------------------------
#
#
# Extract out the level1 from the ObservationContext
level1 = PacsContext(obs.level1)
frames = level1.averaged.getCamera(camera).product.getScience(0)
# if not frames.containsMask('badprobs'):
# print 'Run addgyroprobmask before running this script'
# else:
# ******************************************************************************
# Processing
# ******************************************************************************
# Flag jump/module drop outs
#
frames = scanamorphosMaskLongTermGlitches(frames, stepAfter=20)
#
# Flag calibration block decays
#
frames = scanamorphosBaselinePreprocessing(frames)
#
# Filter on scan speed
frames = runfilteronscanspeed(frames)
if camera == "blue":
# if PhotHelper.isParallelObs(obs):
# pixsize = 3.2
# else:
pixsize = 2.
highpassradius = 15
else:
pixsize = 3.2
highpassradius = 25
frames_hpf = highpassFilter(frames,
highpassradius,
copy=True,
interpolateMaskedValues=True)
System.gc()
#
# spatial deglitching now after just before photProject SPRs PACS-3522 &
# PACS-3906
#s = Sigclip(10, 20)
#s.behavior = Sigclip.CLIP
#s.outliers = Sigclip.BOTH_OUTLIERS
#s.mode = Sigclip.MEDIAN
#
#mdt = MapDeglitchTask()
#mdt(frames, algo = s, deglitchvector="timeordered", calTree=calTree)
#
# save the new glitch mask to be casted back to L1 frames
#if frames.meta.containsKey('Glitchmask') :
# glitchmaskdata = frames.meta["Glitchmask"]
# mask = frames.getMask(glitchmaskdata.value)
#
# Masking the "no scan data" (using BBID 215131301) - Mask name : NoScanData
#frames = photMaskFrames(frames, noScanData = 1, remove=1) #PACS-4008
#
#frames = filterOnScanSpeed(frames,lowScanSpeed=lowscanspeed,highScanSpeed=highscanspeed)
#
#System.gc()
# prior source densities are insufficient to correct on a scan leg basis.
# revisit the following lines if the situation changes.
# scanindices = frames['Status']['ScanLineNumber'].data
# numscanlegs = max(scanindices)
# numtimesteps = len(scanindices)
c1 = LocalStoreContext()
c1_pre = c1.getStoreDir().toString().split('.hcss/')[0]
dir_pre = c1_pre + 'correctscans/'
if not os.path.exists(dir_pre):
print 'Creating directory:'
print dir_pre
print 'for temp storage of scan leg maps'
os.mkdir(dir_pre)
# for i in xrange(20, 22):
# selectindices = scanindices.where(scanindices == i)
# masked pixels have a mask value of True
# onlyselectindices = Bool3d(32, 64, numtimesteps, True)
# onlyselectindices[:,:,selectindices] = False
# frames.removeMask('onescan')
# frames.addMaskType('onescan', 'only the ith scan')
# frames.setMask('onescan', onlyselectindices)
map, mi = photProject(frames_hpf, calTree=calTree, outputPixelsize=pixsize)
coverage = map['coverage'].data
cov_ind = coverage.where(coverage == 0.)
map['image'].data[cov_ind] = Double.NaN
map = centerRaDecMetaData(map)
strobsid = str(obs.obsid)
simpleFitsWriter(product=map['image'],
file=dir_pre + 'scan' + strobsid + camera + '.fits')
#
# Add some Quality information to the frames
frames = addQualityInformation(frames)
#
# Post processing
# cast the glitch mask back into level 1
#if frames.meta.containsKey('Glitchmask') :
# del(frames)
# frames = level1.averaged.getCamera(camera).product.getScience(0)
# frames.removeMask(glitchmaskdata.value)
# frames.addMaskType(glitchmaskdata.value,glitchmaskdata.description)
# frames.setMask(glitchmaskdata.value,mask)
# frames.meta["Glitchmask"] = glitchmaskdata
# level1.averaged.getCamera(camera).product.replace(0, frames)
# obs.level1 = level1
# del(glitchmaskdata, mask)
#
# delete some variables
#del level1, frames, hpfradius1, hpfradius2, outputPixelSize
#del lowscanspeed, highscanspeed, speed, ref, med, index
#del signal_stdev, cutlevel, threshold, HPFmask, s, mdt, image, ad, mi, historyCopy
level1.averaged.getCamera(camera).product.replace(0, frames)
obs.level1 = level1
saveObservation(obs, poolName=poolname, saveCalTree=True)
return
# processing out-focus
if unsynchronized:
before = preprocess(target_dir + '\\out-focus\\before',
out='out',
filename='*')
after = preprocess(target_dir + '\\out-focus\\after',
out='out',
filename='*')
concatenate_files(before, after,
target_dir + '\\out-focus\\merged\\merged.tif')
else:
preprocess(target_dir + '\\out-focus', out='out', filename='*').close()
# processing caspase
if unsynchronized:
path_signal = target_dir + "\\caspase"
path_imp = "\\caspasexy%sc1.tif" % pos
path_imp_out = "\\caspasexy%sc1_sub.tif" % pos
else:
path_signal = target_dir + "\\caspase"
path_imp = "\\seq0001xy%sc1.tif" % pos
path_imp_out = "\\seq0001xy%sc1_sub.tif" % pos
process_caspase_signal(path_signal, path_imp, path_imp_out)
# processing pi signal
process_pi_signal(target_dir, pos, unsynchronized=unsynchronized)
System.gc()
def VesselFinder(channel_array, classifier_path):
channels = channel_array
image = channels[3]
channels = channels[0:3]
proc = image.getProcessor()
directional_op = ImagePlus("directional_op", proc)
tubes = range(5, 130, 12)
img_source = ImagePlus("image", proc)
src = clij2.push(img_source)
dst = clij2.create(src)
sigma = 2
clij2.gaussianBlur2D(src, dst, sigma, sigma)
img_blur2 = clij2.pull(dst)
src.close()
dst.close()
print("Tubeness mt start")
tubenesses = [None] * len(tubes)
rang = range(len(tubes))
threads = []
for i in rang:
threads.append(
threading.Thread(target=run_tube,
args=(img_blur2, tubes[i], i, tubenesses)))
threads[i].start()
[x.join() for x in threads]
print("Tubeness all done")
print(tubenesses)
src = clij2.push(img_source)
dst = clij2.create(src)
sigmas = [5, 20]
imgsigmas = []
for sigma in sigmas:
clij2.gaussianBlur2D(src, dst, sigma, sigma)
img = clij2.pull(dst)
imgsigmas.append(img)
print("Gaussian Blur done")
src.close()
dst.close()
variances = [5, 20]
imgvars = []
for variance in variances:
img = ImagePlus("image", proc)
IJ.run(img, "Variance...", "radius=" + str(variance))
imgvars.append(img)
print("Gaussian Blur done")
featuresArray = FeatureStackArray(image.getStackSize())
stack = ImageStack(image.getWidth(), image.getHeight())
# add new feature here (2/2) and do not forget to add it with a
# unique slice label!
stack.addSlice("directional_op", directional_op.getProcessor())
for i in range(len(sigmas)):
stack.addSlice("sigma" + str(sigmas[i]), imgsigmas[i].getProcessor())
for i in range(len(tubes)):
stack.addSlice("Tubeness" + str(tubes[i]),
tubenesses[i].getProcessor())
for i in range(len(variances)):
stack.addSlice("Variance" + str(variances[i]),
imgvars[i].getProcessor())
for i in range(len(channels)):
stack.addSlice("channel" + str(i + 1), channels[i].getProcessor())
del sigmas
del tubes
del variances
del channels
# create empty feature stack
features = FeatureStack(stack.getWidth(), stack.getHeight(), False)
# set my features to the feature stack
features.setStack(stack)
# put my feature stack into the array
featuresArray.set(features, 0)
featuresArray.setEnabledFeatures(features.getEnabledFeatures())
del stack
wekaSegmentation = WekaSegmentation(image)
wekaSegmentation.setFeatureStackArray(featuresArray)
wekaSegmentation.loadClassifier(classifier_path +
"\\vessel-classifier_big.model")
output = wekaSegmentation.applyClassifier(image, featuresArray, 0, True)
System.gc()
return output
def reset(self, randomize=False):
for n in self.nodes:
n.reset(randomize)
# Force java garbage collection to free (hopefully) unused memory
System.gc()
def windowClosed(self, event): System.gc()
totalee = 0
totalhe = 0
for i in range(nECB):
parCl = colECB.getElementAt(i);
fileOutECaloR.write( '{0:3} {1:4.4f} {2:4.4f} {3:4.4f} {4:4.4f} \n'.format(0,parCl.getPosition()[0],parCl.getPosition()[1],parCl.getPosition()[2],parCl.getEnergy()) )
# totalee += parCl.getEnergy()
for i in range(nHCB):
parCl = colHCB.getElementAt(i);
fileOutHCaloR.write( '{0:3} {1:4.4f} {2:4.4f} {3:4.4f} {4:4.4f} \n'.format(1,parCl.getPosition()[0],parCl.getPosition()[1],parCl.getPosition()[2],parCl.getEnergy()) )
# totalhe += parCl.getEnergy()
# print totalee, totalhe, totalee+totalhe
# if nEvent > 10: break;
del col
del colPF
del colCl
del colTr
del colECB
del colHCB
del evt
reader.close() # close the file
del reader
System.gc()
def run1():
JyNI.JyRefMonitor_setMemDebugFlags(1)
JyWeakReferenceGC.monitorNativeCollection = True
# PyType <- Make native ref-cycle test with heap type to test partly-stub-mode.
# PySet, but no native link to other PyObject
# PyFrozenSet, "
# PyCode, not GC-relevant in CPython
import DemoExtension
#Note:
# For now we attempt to verify JyNI's GC-functionality independently from
# Jython concepts like Jython weak references or Jython GC-module.
# So we use java.lang.ref.WeakReference and java.lang.System.gc to monitor
# and control Java-gc.
#JyNI.JyRefMonitor_setMemDebugFlags(1)
#JyWeakReferenceGC.monitorNativeCollection = True
#l = (123,)
l = ([0, "test"],)
l[0][0] = l
#l = (123, {'a': 0, 'b': "test"})
#l[1]['a'] = l
#We create weak reference to l to monitor collection by Java-GC:
wkl = WeakReference(l)
print "weak(l): "+str(wkl.get())
# We pass down l to some native method. We don't care for the method itself,
# but conversion to native side causes creation of native PyObjects that
# correspond to l and its elements. We will then track the life-cycle of these.
print "make l native..."
DemoExtension.argCountToString(l)
#print "argCountToString.__doc__-address: "+str(JyNI.lookupNativeHandle(DemoExtension.argCountToString.__doc__))
#print "We access a method-doc as this used to cause problems with GC:"
#print " "+DemoExtension.argCountToString.__doc__
#print "Native static objects so far:"
#print JyNI.nativeStaticPyObjectHeads.keySet()
print "Delete l... (but GC not yet ran)"
del l
#l = None
print "weak(l) after del: "+str(wkl.get())
print ""
# monitor.list-methods display the following format:
# [native pointer]{'' | '_GC_J' | '_J'} ([type]) #[native ref-count]: [repr] *[creation time]
# _GC_J means that JyNI tracks the object
# _J means that a JyNI-GC-head exists, but the object is not actually treated by GC
# This can serve monitoring purposes or soft-keep-alive (c.f. java.lang.ref.SoftReference)
# for caching.
print "Leaks before GC:"
monitor.listLeaks()
print ""
# By inserting this line you can confirm that native
# leaks would persist if JyNI-GC is not working:
#JyWeakReferenceGC.nativecollectionEnabled = False
print "calling Java-GC..."
System.gc()
time.sleep(2)
print "weak(l) after GC: "+str(wkl.get())
print ""
monitor.listWouldDeleteNative()
print ""
print "leaks after GC:"
monitor.listLeaks()
print ""
print " "+DemoExtension.argCountToString.__doc__
monitor.listLeaks()
#print "----"
#print monitor.listAll()
print ""
print "===="
print "exit"
print "===="
def reset(self,randomize=False):
for n in self.nodes: n.reset(randomize)
# Force java garbage collection to free (hopefully) unused memory
System.gc()
def run3():
JyNI.JyRefMonitor_setMemDebugFlags(1)
JyWeakReferenceGC.monitorNativeCollection = True
import DemoExtension
#JyNI.JyRefMonitor_setMemDebugFlags(1)
#JyWeakReferenceGC.monitorNativeCollection = True
l = [0, "test1"]
# print l
# DemoExtension.listSetIndex(l, 0, 100.7)
# print l
d = {'a': 7, 'b': "test6"}
#We create weak reference to l to monitor collection by Java-GC:
wkl = WeakReference(l)
wkd = WeakReference(d)
wkl2 = weakref.ref(l)
wkd2 = weakref.ref(d)
#Note:
#=====
#To make this work we'll have to ensure that d gets a GCHead even though
#l doesn't recognize the insertion of d and thus would not explore it.
#In fact every C-stub object must get a GCHead when going native, regardless
#of whether it is inserted somewhere or not, because it might be silently
#inserted and a GCHead is the only way to detect this and fix it.
#Add a call (e.g. to size) to the dict in l on native side to test that
#d still works (which it currently shouldn't).
#Later use d's GCHead to detect the invalid graph and resurrect d's java-part.
#Take care of the GIL here. CStubs must release the GIL when they detect a
#failing backend, so the GC-thread can acquire it and resurrect the backend.
#How to fix the weak reference etc?
#l[0] = d
print "weak(l): "+str(wkl.get())
print "weak(d): "+str(wkd.get())
print "weak2(l): "+str(wkl2())
print "weak2(d): "+str(wkd2())
print "make l native..."
#l[0] = d
#DemoExtension.argCountToString(l)
#l[0] = d
DemoExtension.listSetIndex(l, 0, d)
print "Delete l... (but GC not yet ran)"
#del l
del d
#l = None
#print "weak(l) after del: "+str(wkl.get())
print "weak(d) after del: "+str(wkd.get())
print "weak2(d) after del: "+str(wkd2())
print ""
print "Leaks before GC:"
monitor.listLeaks()
print ""
print "calling Java-GC..."
System.gc()
time.sleep(2)
# if monitor.lastClearGraphValid:
# print "valid graph"
# else:
# print "invalid graph"
# monitor.lastClearGraphValid = False
print "weak(l) after GC: "+str(wkl.get())
#print "l after GC: "+str(l)
print "weak(d) after GC: "+str(wkd.get())
print "weak2(l) after GC: "+str(wkl2())
print "weak2(d) after GC: "+str(wkd2())
wkd = WeakReference(l[0])
print ""
#monitor.listWouldDeleteNative()
print ""
#print "leaks after GC:"
#monitor.listLeaks()
print "l[0], i.e. d after gc:"
#print l[0]
#print len(l[0])
print "------"
print "del l..."
del l
System.gc()
time.sleep(2)
# if monitor.lastClearGraphValid:
# print "valid graph"
# else:
# print "invalid graph"
# monitor.lastClearGraphValid = False
print ""
monitor.listWouldDeleteNative()
print ""
#print "leaks after GC (l deleted):"
#monitor.listLeaks()
#print DemoExtension.argCountToString.__doc__
#monitor.listFreeStatus("dict")
#monitor.listAll()
#GlobalRef.processDelayedCallbacks()
print "weak(d) after GC2: "+str(wkd.get())
print "weak2(l) after GC2: "+str(wkl2())
print "weak2(d) after GC2: "+str(wkd2())
System.gc()
time.sleep(2)
print "weak(d) after GC3: "+str(wkd.get())
monitor.listWouldDeleteNative()
print ""
print "leaks after GC3:"
monitor.listLeaks()
print ""
print "===="
print "exit"
print "===="
def run1():
JyNI.JyRefMonitor_setMemDebugFlags(1)
JyWeakReferenceGC.monitorNativeCollection = True
# PyType <- Make native ref-cycle test with heap type to test partly-stub-mode.
# PySet, but no native link to other PyObject
# PyFrozenSet, "
# PyCode, not GC-relevant in CPython
import DemoExtension
#Note:
# For now we attempt to verify JyNI's GC-functionality independently from
# Jython concepts like Jython weak references or Jython GC-module.
# So we use java.lang.ref.WeakReference and java.lang.System.gc to monitor
# and control Java-gc.
#JyNI.JyRefMonitor_setMemDebugFlags(1)
#JyWeakReferenceGC.monitorNativeCollection = True
#l = (123,)
l = ([0, "test"], )
l[0][0] = l
#l = (123, {'a': 0, 'b': "test"})
#l[1]['a'] = l
#We create weak reference to l to monitor collection by Java-GC:
wkl = WeakReference(l)
print "weak(l): " + str(wkl.get())
# We pass down l to some native method. We don't care for the method itself,
# but conversion to native side causes creation of native PyObjects that
# correspond to l and its elements. We will then track the life-cycle of these.
print "make l native..."
DemoExtension.argCountToString(l)
#print "argCountToString.__doc__-address: "+str(JyNI.lookupNativeHandle(DemoExtension.argCountToString.__doc__))
#print "We access a method-doc as this used to cause problems with GC:"
#print " "+DemoExtension.argCountToString.__doc__
#print "Native static objects so far:"
#print JyNI.nativeStaticPyObjectHeads.keySet()
print "Delete l... (but GC not yet ran)"
del l
#l = None
print "weak(l) after del: " + str(wkl.get())
print ""
# monitor.list-methods display the following format:
# [native pointer]{'' | '_GC_J' | '_J'} ([type]) #[native ref-count]: [repr] *[creation time]
# _GC_J means that JyNI tracks the object
# _J means that a JyNI-GC-head exists, but the object is not actually treated by GC
# This can serve monitoring purposes or soft-keep-alive (c.f. java.lang.ref.SoftReference)
# for caching.
print "Leaks before GC:"
monitor.listLeaks()
print ""
# By inserting this line you can confirm that native
# leaks would persist if JyNI-GC is not working:
#JyWeakReferenceGC.nativecollectionEnabled = False
print "calling Java-GC..."
System.gc()
time.sleep(2)
print "weak(l) after GC: " + str(wkl.get())
print ""
monitor.listWouldDeleteNative()
print ""
print "leaks after GC:"
monitor.listLeaks()
print ""
print " " + DemoExtension.argCountToString.__doc__
monitor.listLeaks()
#print "----"
#print monitor.listAll()
print ""
print "===="
print "exit"
print "===="
def runGC(): System.gc() time.sleep(1)
def run3():
JyNI.JyRefMonitor_setMemDebugFlags(1)
JyWeakReferenceGC.monitorNativeCollection = True
import DemoExtension
#JyNI.JyRefMonitor_setMemDebugFlags(1)
#JyWeakReferenceGC.monitorNativeCollection = True
l = [0, "test1"]
# print l
# DemoExtension.listSetIndex(l, 0, 100.7)
# print l
d = {'a': 7, 'b': "test6"}
#We create weak reference to l to monitor collection by Java-GC:
wkl = WeakReference(l)
wkd = WeakReference(d)
wkl2 = weakref.ref(l)
wkd2 = weakref.ref(d)
#Note:
#=====
#To make this work we'll have to ensure that d gets a GCHead even though
#l doesn't recognize the insertion of d and thus would not explore it.
#In fact every C-stub object must get a GCHead when going native, regardless
#of whether it is inserted somewhere or not, because it might be silently
#inserted and a GCHead is the only way to detect this and fix it.
#Add a call (e.g. to size) to the dict in l on native side to test that
#d still works (which it currently shouldn't).
#Later use d's GCHead to detect the invalid graph and resurrect d's java-part.
#Take care of the GIL here. CStubs must release the GIL when they detect a
#failing backend, so the GC-thread can acquire it and resurrect the backend.
#How to fix the weak reference etc?
#l[0] = d
print "weak(l): " + str(wkl.get())
print "weak(d): " + str(wkd.get())
print "weak2(l): " + str(wkl2())
print "weak2(d): " + str(wkd2())
print "make l native..."
#l[0] = d
#DemoExtension.argCountToString(l)
#l[0] = d
DemoExtension.listSetIndex(l, 0, d)
print "Delete l... (but GC not yet ran)"
#del l
del d
#l = None
#print "weak(l) after del: "+str(wkl.get())
print "weak(d) after del: " + str(wkd.get())
print "weak2(d) after del: " + str(wkd2())
print ""
print "Leaks before GC:"
monitor.listLeaks()
print ""
print "calling Java-GC..."
System.gc()
time.sleep(2)
# if monitor.lastClearGraphValid:
# print "valid graph"
# else:
# print "invalid graph"
# monitor.lastClearGraphValid = False
print "weak(l) after GC: " + str(wkl.get())
#print "l after GC: "+str(l)
print "weak(d) after GC: " + str(wkd.get())
print "weak2(l) after GC: " + str(wkl2())
print "weak2(d) after GC: " + str(wkd2())
wkd = WeakReference(l[0])
print ""
#monitor.listWouldDeleteNative()
print ""
#print "leaks after GC:"
#monitor.listLeaks()
print "l[0], i.e. d after gc:"
#print l[0]
#print len(l[0])
print "------"
print "del l..."
del l
System.gc()
time.sleep(2)
# if monitor.lastClearGraphValid:
# print "valid graph"
# else:
# print "invalid graph"
# monitor.lastClearGraphValid = False
print ""
monitor.listWouldDeleteNative()
print ""
#print "leaks after GC (l deleted):"
#monitor.listLeaks()
#print DemoExtension.argCountToString.__doc__
#monitor.listFreeStatus("dict")
#monitor.listAll()
#GlobalRef.processDelayedCallbacks()
print "weak(d) after GC2: " + str(wkd.get())
print "weak2(l) after GC2: " + str(wkl2())
print "weak2(d) after GC2: " + str(wkd2())
System.gc()
time.sleep(2)
print "weak(d) after GC3: " + str(wkd.get())
monitor.listWouldDeleteNative()
print ""
print "leaks after GC3:"
monitor.listLeaks()
print ""
print "===="
print "exit"
print "===="
