def _getAssociatedImage(self, imageKey):
"""
Get an associated image in PIL format.
:param imageKey: the key of the associated image.
:return: the image in PIL format or None.
"""
info = self._metadata['seriesAssociatedImages'].get(imageKey)
if info is None:
return
series = info['seriesNum']
with self._tileLock:
try:
javabridge.attach()
image = self._bioimage.read(
series=series,
rescale=False, # return internal data types
XYWH=(0, 0, info['sizeX'], info['sizeY']))
except javabridge.JavaException as exc:
es = javabridge.to_string(exc.throwable)
raise TileSourceException('Failed to get Bioformat series (%s, %r).' % (es, (
series, info['sizeX'], info['sizeY'])))
finally:
if javabridge.get_env():
javabridge.detach()
return large_image.tilesource.base._imageToPIL(image)
def write_tile(br, bw, X, Y, Z, C):
"""Read and write one tile of a multichannel image
This function is intended to be run inside of a thread. Since data is sent
to the JVM and the GIL is released, reading images can be performed in
parallel. Writing still requires a lock.
Args:
br ([bfio.BioReader]): BioReader to be read from
bw ([bfio.BioWriter]): BioWriter to be written to
X ([list]): Range of x-values to read/write
Y ([list]): Range of y-values to grab/write
C ([list]): Channel to write
"""
# Connect to the JVM
jutil.attach()
# Read the image
image = br.read_image(X=X, Y=Y, Z=Z)
# Lock the thread and write the image
with lock:
bw.write_image(image, X=[X[0]], Y=[Y[0]], Z=[Z[0]], C=C)
# Detach from the JVM
jutil.detach()
def exit_thread(self):
from bioformats.formatreader import clear_image_reader_cache
self.notify_socket.close()
clear_image_reader_cache()
J.detach()
if self.with_stop_run_loop:
stop_run_loop()
def tearDownModule():
try:
os.environ['PYJNIUS_ACTIVATE']
except KeyError:
import javabridge
javabridge.detach()
javabridge.kill_vm()
def exit_thread(self):
from bioformats.formatreader import clear_image_reader_cache
self.notify_socket.close()
clear_image_reader_cache()
J.detach()
if self.with_stop_run_loop:
stop_run_loop()
def validation_queue_handler():
from javabridge import attach, detach
attach()
try:
while mv_constants.validation_queue_keep_running:
request = mv_constants.validation_queue.get()
if (
not isinstance(request, ValidationRequestController)
or request.cancelled
):
continue
start = time.perf_counter()
try:
validate_module(
request.pipeline, request.module_num, request.callback,
)
except:
pass
# Make sure this thread utilizes less than 1/2 of GIL clock
wait_for = max(0.25, time.perf_counter() - start)
time.sleep(wait_for)
finally:
detach()
logging.info("Exiting the pipeline validation thread")
def tearDownModule():
try:
os.environ["PYJNIUS_ACTIVATE"]
except KeyError:
import javabridge
javabridge.detach()
javabridge.kill_vm()
def __call__(self, id, filename, uri, **kwargs):
"""
Extract metadata from a Datafile using the get_meta function and save
the outputs as DatafileParameters. This function differs from
process_meta in that it generates an output directory in the metadata
store and passes it to the metadata processing func so that outputs
e.g., preview images or metadata files) can be saved.
param id: Datafile ID
type id: integer
param filename: Absolute path to a file for processing
type filename: string
param uri: Dataset URI
type uri: string
param kwargs: Extra arguments
type kwargs: object
return Extracted metadata
rtype dict
"""
_, extension = os.path.splitext(filename)
if extension[1:] not in bioformats.READABLE_FORMATS:
return None
logger.info("Applying Bioformats filter to {}...".format(filename))
# Need to start a JVM in each thread
check_and_start_jvm()
try:
javabridge.attach()
shush_logger()
thumb_rel_path, thumb_abs_path = get_thumbnail_paths(
id, filename, uri)
if not os.path.exists(os.path.dirname(thumb_abs_path)):
os.makedirs(os.path.dirname(thumb_abs_path))
rsp = get_meta(filename, os.path.dirname(thumb_abs_path), **kwargs)
if rsp is not None:
metadata = []
for i in rsp:
metadata.append(self.filter_metadata(i))
return metadata[0]
except Exception as e:
logger.error(str(e))
logger.debug(traceback.format_exc())
finally:
javabridge.detach()
javabridge.kill_vm()
return None
def __del__(self):
if getattr(self, '_bioimage', None) is not None:
try:
javabridge.attach()
self._bioimage.close()
_openImages.remove(self)
finally:
if javabridge.get_env():
javabridge.detach()
def run(self):
javabridge.attach()
try:
self.socket = self.context.socket(zmq.REP)
self.socket.bind(self.address)
logger.info("Binding Knime bridge server to %s" % self.address)
poller = zmq.Poller()
poller.register(self.socket, flags=zmq.POLLIN)
if self.notify_addr is not None:
self.notify_socket = self.context.socket(zmq.SUB)
self.notify_socket.setsockopt(zmq.SUBSCRIBE, "")
self.notify_socket.connect(self.notify_addr)
poller.register(self.notify_socket, flags=zmq.POLLIN)
else:
self.notify_socket = None
start_socket = self.context.socket(zmq.PAIR)
start_socket.connect(self.start_addr)
start_socket.send("OK")
start_socket.close()
try:
while True:
for socket, event in poller.poll(1):
if socket == self.notify_socket:
msg = self.notify_socket.recv_string()
if msg == self.stop_msg:
break
elif socket == self.socket:
msg = self.socket.recv_multipart(copy=False)
session_id_frame = msg.pop(0)
session_id = str(session_id_frame.bytes)
# should be None
wrapper = msg.pop(0)
message_type = msg.pop(0).bytes
if message_type not in self.dispatch:
self.raise_cellprofiler_exception(
session_id,
"Unhandled message type: %s" %
message_type,
)
else:
try:
self.dispatch[message_type](session_id,
message_type,
msg)
except Exception as e:
logger.warn(e.message, exc_info=1)
self.raise_cellprofiler_exception(
session_id, e.message)
else:
continue
break
finally:
if self.notify_socket:
self.notify_socket.close()
self.socket.close()
finally:
javabridge.detach()
def run(self):
javabridge.attach()
try:
self.socket = self.context.socket(zmq.REP)
self.socket.bind(self.address)
logger.info("Binding Knime bridge server to %s" % self.address)
poller = zmq.Poller()
poller.register(self.socket, flags=zmq.POLLIN)
if self.notify_addr is not None:
self.notify_socket = self.context.socket(zmq.SUB)
self.notify_socket.setsockopt(zmq.SUBSCRIBE, "")
self.notify_socket.connect(self.notify_addr)
poller.register(self.notify_socket, flags=zmq.POLLIN)
else:
self.notify_socket = None
start_socket = self.context.socket(zmq.PAIR)
start_socket.connect(self.start_addr)
start_socket.send("OK")
start_socket.close()
try:
while True:
for socket, event in poller.poll(1):
if socket == self.notify_socket:
msg = self.notify_socket.recv_string()
if msg == self.stop_msg:
break
elif socket == self.socket:
msg = self.socket.recv_multipart(copy=False)
session_id_frame = msg.pop(0)
session_id = str(session_id_frame.bytes)
# should be None
wrapper = msg.pop(0)
message_type = msg.pop(0).bytes
if message_type not in self.dispatch:
self.raise_cellprofiler_exception(
session_id,
"Unhandled message type: %s" % message_type)
else:
try:
self.dispatch[message_type](
session_id, message_type, msg)
except Exception, e:
logger.warn(e.message, exc_info=1)
self.raise_cellprofiler_exception(
session_id, e.message)
else:
continue
break
finally:
if self.notify_socket:
self.notify_socket.close()
self.socket.close()
finally:
javabridge.detach()
def run(self):
if p.force_bioformats:
logging.debug("Starting javabridge")
import bioformats
javabridge.start_vm(class_path=bioformats.JARS, run_headless=True)
javabridge.attach()
try:
while 1:
self.tile_collection.cv.acquire()
# If there are no objects in the queue then wait
while not self.tile_collection.loadq:
self.tile_collection.cv.wait()
if self._want_abort:
self.tile_collection.cv.release()
db = DBConnect()
db.CloseConnection()
logging.info('%s aborted' % self.getName())
return
data = heappop(self.tile_collection.loadq)
obKey = data[1]
display_whole_image = data[
2] #display whole image instead of object image
self.tile_collection.cv.release()
# wait until loading has completed before continuing
with self.tile_collection.load_lock:
# Make sure tile hasn't been deleted outside this thread
if not self.tile_collection.tileData.get(obKey, None):
continue
# Get the tile
new_data = imagetools.FetchTile(
obKey, display_whole_image=display_whole_image)
if new_data is None:
#if fetching fails, leave the tile blank
continue
tile_data = self.tile_collection.tileData.get(obKey, None)
# Make sure tile hasn't been deleted outside this thread
if tile_data is not None:
# copy each channel
for i in range(len(tile_data)):
tile_data[i] = new_data[i]
for window in self.notify_window:
wx.PostEvent(window, TileUpdatedEvent(obKey))
finally:
if javabridge.get_env() is not None:
javabridge.detach()
def engine(q_request, q_response):
logger.info("Starting script engine thread")
J.attach()
while True:
try:
msg = J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
q_request.take();""", dict(q_request=q_request))
if logger.level <= logging.INFO:
logger.info("Received engine request: %s", J.to_string(msg))
payload = J.get_collection_wrapper(
J.run_script("msg.payload", dict(msg=msg)))
if J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
msg.command==CPythonScriptEngine.EngineCommands.EXECUTE;
""", dict(msg=msg)):
response = do_execute(payload)
elif J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
msg.command==CPythonScriptEngine.EngineCommands.EVALUATE;
""", dict(msg=msg)):
response = do_evaluate(payload)
elif J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
msg.command==CPythonScriptEngine.EngineCommands.CLOSE_ENGINE;
""", dict(msg=msg)):
logger.info("Exiting script engine thread after close request")
break
else:
logger.warn(
"Received unknown command: %s" %
J.run_script("msg.command.toString()", dict(msg=msg)))
response = J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
var exception = new java.lang.RuntimeException(
java.lang.String.format('Unknown command: %s', msg.command.toString()));
var payload = new java.util.ArrayList();
payload.add(exception);
new CPythonScriptEngine.Message(
CPythonScriptEngine.EngineCommands.EXCEPTION, payload);
""", dict(msg=msg))
J.run_script("q_response.put(response);",
dict(q_response=q_response, response=response))
except:
# To do: how to handle failure, probably from .take()
# Guessing that someone has managed to interrupt our thread
logger.warn("Exiting script engine thread", exc_info=True)
J.detach()
def _monitor_thread():
main_thread = threading.main_thread()
main_thread.join()
if len(_openImages):
try:
javabridge.attach()
while len(_openImages):
source = _openImages.pop()
try:
source._bioimage.close()
except Exception:
pass
source._bioimage = None
finally:
if javabridge.get_env():
javabridge.detach()
_stopJavabridge()
def run(self):
javabridge.attach()
try:
while 1:
self.tile_collection.cv.acquire()
# If there are no objects in the queue then wait
while not self.tile_collection.loadq:
self.tile_collection.cv.wait()
if self._want_abort:
self.tile_collection.cv.release()
logging.info('%s aborted' % self.getName())
return
data = heappop(self.tile_collection.loadq)
obKey = data[1]
display_whole_image = data[
2] #display whole image instead of object image
self.tile_collection.cv.release()
# wait until loading has completed before continuing
with self.tile_collection.load_lock:
# Make sure tile hasn't been deleted outside this thread
if not self.tile_collection.tileData.get(obKey, None):
continue
# Get the tile
new_data = imagetools.FetchTile(
obKey, display_whole_image=display_whole_image)
if new_data is None:
#if fetching fails, leave the tile blank
continue
tile_data = self.tile_collection.tileData.get(obKey, None)
# Make sure tile hasn't been deleted outside this thread
if tile_data is not None:
# copy each channel
for i in range(len(tile_data)):
tile_data[i] = new_data[i]
wx.PostEvent(self.notify_window,
TileUpdatedEvent(obKey))
finally:
javabridge.detach()
def run(self):
javabridge.attach()
try:
while 1:
self.tile_collection.cv.acquire()
# If there are no objects in the queue then wait
while not self.tile_collection.loadq:
self.tile_collection.cv.wait()
if self._want_abort:
self.tile_collection.cv.release()
logging.info("%s aborted" % self.getName())
return
data = heappop(self.tile_collection.loadq)
obKey = data[1]
display_whole_image = data[2] # display whole image instead of object image
self.tile_collection.cv.release()
# wait until loading has completed before continuing
with self.tile_collection.load_lock:
# Make sure tile hasn't been deleted outside this thread
if not self.tile_collection.tileData.get(obKey, None):
continue
# Get the tile
new_data = imagetools.FetchTile(obKey, display_whole_image=display_whole_image)
if new_data is None:
# if fetching fails, leave the tile blank
continue
tile_data = self.tile_collection.tileData.get(obKey, None)
# Make sure tile hasn't been deleted outside this thread
if tile_data is not None:
# copy each channel
for i in range(len(tile_data)):
tile_data[i] = new_data[i]
wx.PostEvent(self.notify_window, TileUpdatedEvent(obKey))
finally:
javabridge.detach()
def fn():
from bioformats import load_image_url
import javabridge
from scipy.io.matlab.mio import loadmat
from cellprofiler.modules.loadimages import url2pathname
javabridge.attach()
with self.image_dict_lock:
generation = self.image_dict_generation
for k, v in well.iteritems():
sd = {}
with self.image_dict_lock:
if self.image_dict_generation > generation:
return
self.image_dict[k] = sd
for c, fd in enumerate(v):
if PlateData.D_CHANNEL in fd:
channel = fd[PlateData.D_CHANNEL]
else:
channel = str(c + 1)
url = fd[PlateData.D_FILENAME]
try:
if url.lower().endswith(".mat"):
img = loadmat(
url2pathname(url),
struct_as_record=True)["Image"]
else:
img = load_image_url(url)
with self.image_dict_lock:
if self.image_dict_generation > generation:
return
sd[channel] = img
except:
traceback.print_exc()
pass
wx.CallAfter(self.update_figure)
javabridge.static_call(
"java/lang/System", "gc", "()V")
javabridge.detach()
def fn():
from bioformats import load_image_url
import javabridge
from scipy.io.matlab.mio import loadmat
from cellprofiler.modules.loadimages import url2pathname
javabridge.attach()
with self.image_dict_lock:
generation = self.image_dict_generation
for k, v in well.iteritems():
sd = {}
with self.image_dict_lock:
if self.image_dict_generation > generation:
return
self.image_dict[k] = sd
for c, fd in enumerate(v):
if PlateData.D_CHANNEL in fd:
channel = fd[PlateData.D_CHANNEL]
else:
channel = str(c+1)
url = fd[PlateData.D_FILENAME]
try:
if url.lower().endswith(".mat"):
img = loadmat(
url2pathname(url),
struct_as_record=True)["Image"]
else:
img = load_image_url(url)
with self.image_dict_lock:
if self.image_dict_generation > generation:
return
sd[channel] = img
except:
traceback.print_exc()
pass
wx.CallAfter(self.update_figure)
javabridge.static_call(
"java/lang/System", "gc", "()V")
javabridge.detach()
def engine_requester():
J.attach()
while True:
try:
msg = J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
CPythonScriptEngine.engineRequestQueue.take();""")
if logger.level <= logging.INFO:
logger.info("Received engine request: %s", J.to_string(msg))
payload = J.get_collection_wrapper(
J.run_script("msg.payload", dict(msg=msg)))
if J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
msg.command==CPythonScriptEngine.EngineCommands.NEW_ENGINE;
""", dict(msg=msg)):
do_new_engine(payload)
elif J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
msg.command==CPythonScriptEngine.EngineCommands.CLOSE_SERVICE;
""", dict(msg=msg)):
logger.info("Exiting script service thread in response to "
"termination request")
break
else:
J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
var exception = new java.lang.RuntimeException(
java.lang.String.format('Unknown command: %s', msg.command.toString()));
var payload = new java.util.ArrayList();
payload.add(exception);
var response = new CPythonScriptEngine.Message(
CPythonScriptEngine.EngineCommands.EXCEPTION, payload);
CPythonScriptEngine.engineResponseQueue.put(response);
""")
except:
# To do: how to handle failure, probably from .take()
# Guessing that someone has managed to interrupt our thread
logger.warn("Exiting script service thread", exc_info=True)
J.detach()
def detach(self):
javabridge.detach()
def __init__(self, df, penaltydiscount = 4, depth = 3, faithfulness = True, numofthreads = 2, verbose = False, java_max_heap_size = None):
tetrad_libdir = os.path.join(os.path.dirname(__file__), 'lib')
for l in glob.glob(tetrad_libdir + os.sep + "*.jar"):
javabridge.JARS.append(str(l))
javabridge.start_vm(run_headless=True, max_heap_size = java_max_heap_size)
javabridge.attach()
tetradData = None
if(len(df.index)*df.columns.size <= 1500):
node_list = javabridge.JClassWrapper('java.util.ArrayList')()
for col in df.columns:
nodi = javabridge.JClassWrapper('edu.cmu.tetrad.data.ContinuousVariable')(col)
node_list.add(nodi)
dataBox = javabridge.JClassWrapper('edu.cmu.tetrad.data.DoubleDataBox')(len(df.index),df.columns.size)
for row in df.index:
for col in range(0,df.columns.size):
value = javabridge.JClassWrapper('java.lang.Double')(df.ix[row][col])
dataBox.set(row,col,value)
tetradData = javabridge.JClassWrapper('edu.cmu.tetrad.data.BoxDataSet')(dataBox, node_list)
else:
#Generate random name
temp_data_file = ''.join(random.choice(string.lowercase) for i in range(10)) + '.csv'
temp_data_path = os.path.join(tempfile.gettempdir(), temp_data_file)
df.to_csv(temp_data_path, sep = '\t', index = False)
excludeVar = javabridge.JClassWrapper('java.util.HashSet')()
excludeVar.add('MULT')
# Read in method 1
f = javabridge.JClassWrapper('java.nio.file.Paths').get(temp_data_path)
dataReader = javabridge.JClassWrapper('edu.cmu.tetrad.io.TabularContinuousDataReader')(f,'\t')
tetradData = dataReader.readInData(excludeVar)
# Read in method 2 -- Depreciated
# f = javabridge.JClassWrapper('java.io.File')(temp_data_path)
# tetradData = javabridge.JClassWrapper('edu.cmu.tetrad.data.BigDataSetUtility').readInContinuousData(f,'\t',excludeVar)
os.remove(temp_data_path)
tetradData = javabridge.JClassWrapper('edu.cmu.tetrad.data.CovarianceMatrixOnTheFly')(tetradData)
score = javabridge.JClassWrapper('edu.cmu.tetrad.search.SemBicScore')(tetradData)
score.setPenaltyDiscount(penaltydiscount) # set to 2 if variable# <= 50 otherwise set it to 4
fgs = javabridge.JClassWrapper('edu.cmu.tetrad.search.Fgs')(score)
fgs.setDepth(depth)#-1
fgs.setNumPatternsToStore(0)
fgs.setFaithfulnessAssumed(faithfulness)
fgs.setParallelism(numofthreads)
fgs.setVerbose(verbose)
tetradGraph = fgs.search()
graph = pydot.Dot(graph_type='digraph')
n = tetradGraph.getNodeNames().toString()
n = n[1:len(n)-1]
n = n.split(",")
nodes = []
for i in range(0,len(n)):
node = n[i]
n[i] = node.strip()
nodes.append(pydot.Node(n[i]))
graph.add_node(nodes[i])
self.nodes = n
e = tetradGraph.getEdges().toString()
e = e[1:len(e)-1]
e = e.split(",")
for i in range(0,len(e)):
e[i] = e[i].strip()
token = e[i].split(" ")
if(len(token) == 3):
src = token[0]
arc = token[1]
dst = token[2]
if(pycausal.isNodeExisting(n,src) and pycausal.isNodeExisting(n,dst)):
edge = pydot.Edge(nodes[n.index(src)],nodes[n.index(dst)])
if(arc == "---"):
edge.set_arrowhead("none")
graph.add_edge(edge)
self.edges = e
javabridge.detach()
javabridge.kill_vm()
self.graph = graph
def tearDownClass(cls):
J.detach()
def tearDown(self):
J.detach()
def run(self):
javabridge.attach()
try:
main(sys.argv)
finally:
javabridge.detach()
def tearDown(self):
javabridge.detach()
def run(my_integer=my_integer):
env = javabridge.attach()
self.assertEqual(my_integer.intValue(), my_value)
del my_integer
javabridge.detach()
def stop_vm(self):
javabridge.detach()
javabridge.kill_vm()
def get_user_loader():
'''The class loader used to load user plugins'''
return J.static_call("ij/IJ", "getClassLoader", "()Ljava/lang/ClassLoader;")
def get_plugin(classname):
'''Return an instance of the named plugin'''
if classname.startswith("ij."):
cls = J.class_for_name(classname)
else:
cls = J.class_for_name(classname, get_user_loader())
cls = J.get_class_wrapper(cls, True)
constructor = J.get_constructor_wrapper(cls.getConstructor(None))
return constructor.newInstance(None)
if __name__=="__main__":
import sys
J.attach()
try:
commands = get_commands()
print "Commands: "
for command in commands:
print "\t" + command
if len(sys.argv) == 2:
execute_command(sys.argv[1])
elif len(sys.argv) > 2:
execute_command(sys.argv[1], sys.argv[2])
finally:
J.detach()
J.kill_vm()
def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs):
self._xyzInRange(x, y, z)
ft = fc = fz = 0
fseries = self._metadata['frameSeries'][0]
if kwargs.get('frame') is not None:
frame = int(kwargs.get('frame'))
fc = frame % self._metadata['sizeC']
fz = (frame // self._metadata['sizeC']) % self._metadata['sizeZ']
ft = (frame // self._metadata['sizeC'] //
self._metadata['sizeZ']) % self._metadata['sizeT']
fxy = (frame // self._metadata['sizeC'] //
self._metadata['sizeZ'] // self._metadata['sizeT'])
if frame < 0 or fxy > self._metadata['sizeXY']:
raise TileSourceException('Frame does not exist')
fseries = self._metadata['frameSeries'][fxy]
seriesLevel = self.levels - 1 - z
scale = 1
while seriesLevel >= len(fseries['series']):
seriesLevel -= 1
scale *= 2
offsetx = x * self.tileWidth * scale
offsety = y * self.tileHeight * scale
width = min(self.tileWidth * scale, self.sizeX // 2 ** seriesLevel - offsetx)
height = min(self.tileHeight * scale, self.sizeY // 2 ** seriesLevel - offsety)
sizeXAtScale = fseries['sizeX'] // (2 ** seriesLevel)
sizeYAtScale = fseries['sizeY'] // (2 ** seriesLevel)
finalWidth = width // scale
finalHeight = height // scale
width = min(width, sizeXAtScale - offsetx)
height = min(height, sizeYAtScale - offsety)
with self._tileLock:
try:
javabridge.attach()
if width > 0 and height > 0:
tile = self._bioimage.read(
c=fc, z=fz, t=ft, series=fseries['series'][seriesLevel],
rescale=False, # return internal data types
XYWH=(offsetx, offsety, width, height))
else:
# We need the same dtype, so read 1x1 at 0x0
tile = self._bioimage.read(
c=fc, z=fz, t=ft, series=fseries['series'][seriesLevel],
rescale=False, # return internal data types
XYWH=(0, 0, 1, 1))
tile = numpy.zeros(tuple([0, 0] + list(tile.shape[2:])), dtype=tile.dtype)
format = TILE_FORMAT_NUMPY
except javabridge.JavaException as exc:
es = javabridge.to_string(exc.throwable)
raise TileSourceException('Failed to get Bioformat region (%s, %r).' % (es, (
fc, fz, ft, fseries, self.sizeX, self.sizeY, offsetx, offsety, width, height)))
finally:
if javabridge.get_env():
javabridge.detach()
if scale > 1:
tile = tile[::scale, ::scale]
if tile.shape[:2] != (finalHeight, finalWidth):
fillValue = 0
if tile.dtype == numpy.uint16:
fillValue = 65535
elif tile.dtype == numpy.uint8:
fillValue = 255
elif tile.dtype.kind == 'f':
fillValue = 1
retile = numpy.full(
tuple([finalHeight, finalWidth] + list(tile.shape[2:])),
fillValue,
dtype=tile.dtype)
retile[0:min(tile.shape[0], finalHeight), 0:min(tile.shape[1], finalWidth)] = tile[
0:min(tile.shape[0], finalHeight), 0:min(tile.shape[1], finalWidth)]
tile = retile
return self._outputTile(tile, format, x, y, z, pilImageAllowed, numpyAllowed, **kwargs)
def __init__(self, df, structurePrior = 1.0, samplePrior = 1.0, depth = 3, faithfulness = True, numofthreads = 2, verbose = False, java_max_heap_size = None):
tetrad_libdir = os.path.join(os.path.dirname(__file__), 'lib')
for l in glob.glob(tetrad_libdir + os.sep + "*.jar"):
javabridge.JARS.append(str(l))
javabridge.start_vm(run_headless=True, max_heap_size = java_max_heap_size)
javabridge.attach()
score = None
# if(len(df.index)*df.columns.size <= 1500):
dataBox = javabridge.JClassWrapper('edu.cmu.tetrad.data.VerticalIntDataBox')(len(df.index),df.columns.size)
node_list = javabridge.JClassWrapper('java.util.ArrayList')()
col_no = 0
for col in df.columns:
cat_array = sorted(set(df[col]))
cat_list = javabridge.JClassWrapper('java.util.ArrayList')()
for cat in cat_array:
catname = javabridge.JClassWrapper('java.lang.String')(cat)
cat_list.add(catname)
nodname = javabridge.JClassWrapper('java.lang.String')(col)
nodi = javabridge.JClassWrapper('edu.cmu.tetrad.data.DiscreteVariable')(nodname,cat_list)
node_list.add(nodi)
for row in df.index:
value = javabridge.JClassWrapper('java.lang.Integer')(cat_array.index(df.ix[row][col_no]))
dataBox.set(row,col_no,value)
col_no = col_no + 1
tetradData = javabridge.JClassWrapper('edu.cmu.tetrad.data.BoxDataSet')(dataBox, node_list)
# else:
# Generate random name
# temp_data_file = ''.join(random.choice(string.lowercase) for i in range(10)) + '.csv'
# temp_data_path = os.path.join(tempfile.gettempdir(), temp_data_file)
# df.to_csv(temp_data_path, sep = "\t", index = False)
# excludeVar = javabridge.JWrapper(javabridge.make_instance("java/util/HashSet","()V"))
# excludeVar.add("MULT")
# Read in method 1
# f = javabridge.static_call("java/nio/file/Paths","get","(Ljava/lang/String;)V",temp_data_path)
# dataReader = javabridge.make_instance("edu/cmu/tetrad/io/TabularContinuousDataReader","(Ljava/nio/file/Path;C)V",f,"\t")
# dataReader = javabridge.JWrapper(dataReader)
# tetradData = dataReader.readInData(excludeVar)
# Read in method 2 -- Depreciated
# f = javabridge.make_instance("java/io/File", "(Ljava/lang/String;)V",temp_data_path)
# tetradData = javabridge.static_call("edu/cmu/tetrad/data/BigDataSetUtility", "readInDiscreteData", "(Ljava/io/File;CLjava/util/Set;)Ledu/cmu/tetrad/data/DataSet;",f,"\t",excludeVar)
# os.remove(temp_data_path)
score = javabridge.JClassWrapper('edu.cmu.tetrad.search.BDeuScore')(tetradData)
score.setStructurePrior(structurePrior)
score.setSamplePrior(samplePrior)
fgs = javabridge.JClassWrapper('edu.cmu.tetrad.search.Fgs')(score)
fgs.setDepth(depth)
fgs.setNumPatternsToStore(0)
fgs.setFaithfulnessAssumed(faithfulness)
fgs.setParallelism(numofthreads)
fgs.setVerbose(verbose)
tetradGraph = fgs.search()
graph = pydot.Dot(graph_type='digraph')
n = tetradGraph.getNodeNames().toString()
n = n[1:len(n)-1]
n = n.split(",")
nodes = []
for i in range(0,len(n)):
node = n[i]
n[i] = node.strip()
nodes.append(pydot.Node(n[i]))
graph.add_node(nodes[i])
self.nodes = n
e = tetradGraph.getEdges().toString()
e = e[1:len(e)-1]
e = e.split(",")
for i in range(0,len(e)):
e[i] = e[i].strip()
token = e[i].split(" ")
if(len(token) == 3):
src = token[0]
arc = token[1]
dst = token[2]
if(pycausal.isNodeExisting(n,src) and pycausal.isNodeExisting(n,dst)):
edge = pydot.Edge(nodes[n.index(src)],nodes[n.index(dst)])
if(arc == "---"):
edge.set_arrowhead("none")
graph.add_edge(edge)
self.edges = e
javabridge.detach()
javabridge.kill_vm()
self.graph = graph
def tearDownClass(self):
J.detach()
def stop_vm(self):
javabridge.detach()
javabridge.kill_vm()
def process_meta(df_id, schema_name, overwrite=False, **kwargs):
"""Extract metadata from a Datafile using the get_meta function and save the
outputs as DatafileParameters.
Parameters
----------
df_id: int
ID of Datafile instance to process.
schema_name: str
Names of schema which describes ParameterNames
add: boolean (default: False)
Specifies whether or not to add to an existing Parameterset for this
Datafile rather that overwriting or exiting.
overwrite: boolean (default: False)
Specifies whether to overwrite any exisiting parametersets for
this datafile.
Returns
-------
None
"""
from .metadata import get_meta
if acquire_datafile_lock(df_id):
# Need to start a JVM in each thread
check_and_start_jvm()
try:
javabridge.attach()
log4j.basic_config()
schema = Schema.objects.get(namespace__exact=schema_name)
df = DataFile.objects.get(id=df_id)
if DatafileParameterSet.objects\
.filter(schema=schema, datafile=df).exists():
if overwrite:
psets = DatafileParameterSet.objects.get(schema=schema,
datafile=df)
logger.warning("Overwriting parametersets for %s",
df.filename)
for ps in psets:
delete_old_parameterset(ps)
else:
logger.warning("Parametersets for %s already exist.",
df.filename)
return
dfo = DataFileObject.objects.filter(datafile__id=df.id,
verified=True).first()
input_file_path = dfo.get_full_path()
logger.debug("Processing file: %s", input_file_path)
metadata_params = get_meta(input_file_path, **kwargs)
if not metadata_params:
logger.debug("No metadata to save")
return
for sm in metadata_params:
ps = DatafileParameterSet(schema=schema, datafile=df)
ps.save()
logger.debug("Saving parameters for: %s", input_file_path)
save_parameters(schema, ps, sm)
except Exception as err:
logger.exception(err)
finally:
release_datafile_lock(df_id)
javabridge.detach()
def process_meta_file_output(df_id, schema_name, overwrite=False, **kwargs):
"""Extract metadata from a Datafile using the get_meta function and save the
outputs as DatafileParameters. This function differs from process_meta in
that it generates an output directory in the metadata store and passes it
to the metadata processing func so that outputs (e.g., preview images or
metadata files) can be saved.
Parameters
----------
df_id: int
ID of Datafile instance to process.
schema_name: str
Names of schema which describes ParameterNames
add: Boolean (default: False)
Specifies whether or not to add to an existing Parameterset for this
Datafile rather that overwriting or exiting.
overwrite: Boolean (default: False)
Specifies whether to overwrite any exisiting parametersets for
this datafile.
Returns
-------
None
"""
from .metadata import get_meta
if acquire_datafile_lock(df_id):
# Need to start a JVM in each thread
check_and_start_jvm()
try:
javabridge.attach()
log4j.basic_config()
schema = Schema.objects.get(namespace__exact=schema_name)
df = DataFile.objects.get(id=df_id)
if DatafileParameterSet.objects\
.filter(schema=schema, datafile=df).exists():
if overwrite:
psets = DatafileParameterSet.objects.get(schema=schema,
datafile=df)
logger.warning("Overwriting parametersets for %s",
df.filename)
for ps in psets:
delete_old_parameterset(ps)
else:
logger.warning("Parametersets for %s already exist.",
df.filename)
return
dfo = DataFileObject.objects.filter(datafile__id=df.id,
verified=True).first()
input_file_path = dfo.get_full_path()
output_rel_path = os.path.join(
os.path.dirname(urlparse.urlparse(dfo.uri).path), str(df.id))
output_path = os.path.join(settings.METADATA_STORE_PATH,
output_rel_path)
if not os.path.exists(output_path):
os.makedirs(output_path)
logger.debug("Processing file: %s" % input_file_path)
metadata_params = get_meta(input_file_path, output_path, **kwargs)
if not metadata_params:
logger.debug("No metadata to save")
return
for sm in metadata_params:
ps = DatafileParameterSet(schema=schema, datafile=df)
ps.save()
logger.debug("Saving parameters for: %s", input_file_path)
save_parameters(schema, ps, sm)
except Exception as err:
logger.exception(err)
finally:
release_datafile_lock(df_id)
javabridge.detach()
metadata_params = func(input_file_path, output_path, **kwargs)
if not metadata_params:
logger.debug("No metadata to save")
return
for sm in metadata_params:
ps = DatafileParameterSet(schema=schema, datafile=df)
ps.save()
logger.debug("Saving parameters for: %s" % input_file_path)
save_parameters(schema, ps, sm)
except Exception, e:
logger.debug(e)
finally:
release_datafile_lock(df.id)
javabridge.detach()
@task(name="mytardisbf.process_meta",
ignore_result=True)
def process_meta(func, df, schema_name, overwrite=False, **kwargs):
"""Extract metadata from a Datafile using a provided function and save the
outputs as DatafileParameters.
Parameters
----------
func: Function
Function to extract metadata from a file. Function must have
input_file_path as an argument e.g.:
def meta_proc(input_file_path, **kwargs):
...
def tearDownClass(self):
J.detach()
def exit_thread(self):
from bioformats.formatreader import clear_image_reader_cache
self.notify_socket.close()
clear_image_reader_cache()
J.detach()
def exit_thread(self):
from bioformats.formatreader import clear_image_reader_cache
self.notify_socket.close()
clear_image_reader_cache()
J.detach()
print('big endian:', meta.getPixelsBigEndian(0, 0))
print('dim order:', meta.getPixelsDimensionOrder(0, 0))
print('pixel type:', meta.getPixelsPixelType(0, 0))
print('size x:', meta.getPixelsSizeX(0, 0))
print('size y:', meta.getPixelsSizeY(0, 0))
print('size c:', meta.getPixelsSizeC(0, 0))
print('size z:', meta.getPixelsSizeZ(0, 0))
print('size t:', meta.getPixelsSizeT(0, 0))
print('samples per pixel:', meta.getLogicalChannelSamplesPerPixel(0, 0))
writer.setMetadataRetrieve(meta)
writer.setId(out_file)
for image in images:
if len(image.shape)==3 and image.shape[2] == 3:
save_im = np.array([image[:,:,0], image[:,:,1], image[:,:,2]]).astype(np.uint8).flatten()
else:
save_im = image.astype(np.uint8).flatten()
writer.saveBytes(env.make_byte_array(save_im), (image is images[-1]))
writer.close()
print('Done writing image :)')
# import PIL.Image as Image
# im = Image.open(out_file, 'r')
# im.show()
jutil.detach()
app.MainLoop()
def run(my_integer = my_integer):
env = javabridge.attach()
self.assertEqual(my_integer.intValue(),my_value)
del my_integer
javabridge.detach()
def interface(self,
start_signal,
image_set_start=1,
image_set_end=None,
overwrite=True):
'''Top-half thread for running an analysis. Sets up grouping for jobs,
deals with returned measurements, reports status periodically.
start_signal- signal this semaphore when jobs are ready.
image_set_start - beginning image set number to process
image_set_end - last image set number to process
overwrite - whether to recompute imagesets that already have data in initial_measurements.
'''
from javabridge import attach, detach
posted_analysis_started = False
acknowledged_thread_start = False
measurements = None
workspace = None
attach()
try:
# listen for pipeline events, and pass them upstream
self.pipeline.add_listener(lambda pipe, evt: self.post_event(evt))
initial_measurements = None
if self.output_path is None:
# Caller wants a temporary measurements file.
fd, filename = tempfile.mkstemp(
".h5", dir=cpprefs.get_temporary_directory())
try:
fd = os.fdopen(fd, "wb")
fd.write(self.initial_measurements_buf)
fd.close()
initial_measurements = cpmeas.Measurements(
filename=filename, mode="r")
measurements = cpmeas.Measurements(
image_set_start=None,
copy=initial_measurements,
mode="a")
finally:
if initial_measurements is not None:
initial_measurements.close()
os.unlink(filename)
else:
with open(self.output_path, "wb") as fd:
fd.write(self.initial_measurements_buf)
measurements = cpmeas.Measurements(image_set_start=None,
filename=self.output_path,
mode="a")
# The shared dicts are needed in jobserver()
self.shared_dicts = [m.get_dictionary() for m in self.pipeline.modules()]
workspace = cpw.Workspace(self.pipeline, None, None, None,
measurements, cpimage.ImageSetList())
if image_set_end is None:
image_set_end = measurements.get_image_numbers()[-1]
image_sets_to_process = filter(
lambda x: x >= image_set_start and x <= image_set_end,
measurements.get_image_numbers())
self.post_event(AnalysisStarted())
posted_analysis_started = True
# reset the status of every image set that needs to be processed
has_groups = measurements.has_groups()
if self.pipeline.requires_aggregation():
overwrite = True
if has_groups and not overwrite:
if not measurements.has_feature(cpmeas.IMAGE, self.STATUS):
overwrite = True
else:
group_status = {}
for image_number in measurements.get_image_numbers():
group_number = measurements[
cpmeas.IMAGE, cpmeas.GROUP_NUMBER, image_number]
status = measurements[cpmeas.IMAGE, self.STATUS,
image_number]
if status != self.STATUS_DONE:
group_status[group_number] = self.STATUS_UNPROCESSED
elif group_number not in group_status:
group_status[group_number] = self.STATUS_DONE
new_image_sets_to_process = []
for image_set_number in image_sets_to_process:
needs_reset = False
if (overwrite or
(not measurements.has_measurements(
cpmeas.IMAGE, self.STATUS, image_set_number)) or
(measurements[cpmeas.IMAGE, self.STATUS, image_set_number]
!= self.STATUS_DONE)):
needs_reset = True
elif has_groups:
group_number = measurements[
cpmeas.IMAGE, cpmeas.GROUP_NUMBER, image_set_number]
if group_status[group_number] != self.STATUS_DONE:
needs_reset = True
if needs_reset:
measurements[cpmeas.IMAGE, self.STATUS, image_set_number] = \
self.STATUS_UNPROCESSED
new_image_sets_to_process.append(image_set_number)
image_sets_to_process = new_image_sets_to_process
# Find image groups. These are written into measurements prior to
# analysis. Groups are processed as a single job.
if has_groups or self.pipeline.requires_aggregation():
worker_runs_post_group = True
job_groups = {}
for image_set_number in image_sets_to_process:
group_number = measurements[cpmeas.IMAGE,
cpmeas.GROUP_NUMBER,
image_set_number]
group_index = measurements[cpmeas.IMAGE,
cpmeas.GROUP_INDEX,
image_set_number]
job_groups[group_number] = job_groups.get(group_number, []) + [(group_index, image_set_number)]
job_groups = [[isn for _, isn in sorted(job_groups[group_number])]
for group_number in sorted(job_groups)]
else:
worker_runs_post_group = False # prepare_group will be run in worker, but post_group is below.
job_groups = [[image_set_number] for image_set_number in image_sets_to_process]
# XXX - check that any constructed groups are complete, i.e.,
# image_set_start and image_set_end shouldn't carve them up.
if not worker_runs_post_group:
# put the first job in the queue, then wait for the first image to
# finish (see the check of self.finish_queue below) to post the rest.
# This ensures that any shared data from the first imageset is
# available to later imagesets.
self.work_queue.put((job_groups[0],
worker_runs_post_group,
True))
waiting_for_first_imageset = True
del job_groups[0]
else:
waiting_for_first_imageset = False
for job in job_groups:
self.work_queue.put((job, worker_runs_post_group, False))
job_groups = []
start_signal.release()
acknowledged_thread_start = True
# We loop until every image is completed, or an outside event breaks the loop.
while not self.cancelled:
# gather measurements
while not self.received_measurements_queue.empty():
image_numbers, buf = self.received_measurements_queue.get()
image_numbers = [int(i) for i in image_numbers]
recd_measurements = cpmeas.load_measurements_from_buffer(buf)
self.copy_recieved_measurements(recd_measurements, measurements, image_numbers)
recd_measurements.close()
del recd_measurements
# check for jobs in progress
while not self.in_process_queue.empty():
image_set_numbers = self.in_process_queue.get()
for image_set_number in image_set_numbers:
measurements[cpmeas.IMAGE, self.STATUS, int(image_set_number)] = self.STATUS_IN_PROCESS
# check for finished jobs that haven't returned measurements, yet
while not self.finished_queue.empty():
finished_req = self.finished_queue.get()
measurements[
cpmeas.IMAGE, self.STATUS, int(finished_req.image_set_number)] = self.STATUS_FINISHED_WAITING
if waiting_for_first_imageset:
assert isinstance(finished_req,
ImageSetSuccessWithDictionary)
self.shared_dicts = finished_req.shared_dicts
waiting_for_first_imageset = False
assert len(self.shared_dicts) == len(self.pipeline.modules())
# if we had jobs waiting for the first image set to finish,
# queue them now that the shared state is available.
for job in job_groups:
self.work_queue.put((job, worker_runs_post_group, False))
finished_req.reply(Ack())
# check progress and report
counts = collections.Counter(measurements[cpmeas.IMAGE, self.STATUS, image_set_number]
for image_set_number in image_sets_to_process)
self.post_event(AnalysisProgress(counts))
# Are we finished?
if counts[self.STATUS_DONE] == len(image_sets_to_process):
last_image_number = measurements.get_image_numbers()[-1]
measurements.image_set_number = last_image_number
if not worker_runs_post_group:
self.pipeline.post_group(workspace, {})
workspace = cpw.Workspace(self.pipeline,
None, None, None,
measurements, None, None)
workspace.post_run_display_handler = \
self.post_run_display_handler
self.pipeline.post_run(workspace)
break
measurements.flush()
# not done, wait for more work
with self.interface_work_cv:
while (self.paused or
((not self.cancelled) and
self.in_process_queue.empty() and
self.finished_queue.empty() and
self.received_measurements_queue.empty())):
self.interface_work_cv.wait() # wait for a change of status or work to arrive
finally:
detach()
# Note - the measurements file is owned by the queue consumer
# after this post_event.
#
if not acknowledged_thread_start:
start_signal.release()
if posted_analysis_started:
was_cancelled = self.cancelled
self.post_event(AnalysisFinished(measurements, was_cancelled))
self.stop_workers()
self.analysis_id = False # this will cause the jobserver thread to exit
def tearDownClass(cls):
J.detach()
print('big endian:', meta.getPixelsBigEndian(0, 0))
print('dim order:', meta.getPixelsDimensionOrder(0, 0))
print('pixel type:', meta.getPixelsPixelType(0, 0))
print('size x:', meta.getPixelsSizeX(0, 0))
print('size y:', meta.getPixelsSizeY(0, 0))
print('size c:', meta.getPixelsSizeC(0, 0))
print('size z:', meta.getPixelsSizeZ(0, 0))
print('size t:', meta.getPixelsSizeT(0, 0))
print('samples per pixel:', meta.getLogicalChannelSamplesPerPixel(0, 0))
writer.setMetadataRetrieve(meta)
writer.setId(out_file)
for image in images:
if len(image.shape) == 3 and image.shape[2] == 3:
save_im = np.array(
[image[:, :, 0], image[:, :, 1],
image[:, :, 2]]).astype(np.uint8).flatten()
else:
save_im = image.astype(np.uint8).flatten()
writer.saveBytes(env.make_byte_array(save_im), (image is images[-1]))
writer.close()
print('Done writing image :)')
# import PIL.Image as Image
# im = Image.open(out_file, 'r')
# im.show()
jutil.detach()
app.MainLoop()
def tearDown(self):
javabridge.detach()
def __init__(self, path, **kwargs): # noqa
"""
Initialize the tile class. See the base class for other available
parameters.
:param path: the associated file path.
:param maxSize: either a number or an object with {'width': (width),
'height': height} in pixels. If None, the default max size is
used.
"""
super().__init__(path, **kwargs)
self._logger = config.getConfig('logger')
largeImagePath = self._getLargeImagePath()
ext = os.path.splitext(largeImagePath)[1]
if not ext:
raise TileSourceException(
'File cannot be opened via bioformats because it has no '
'extension to specify the file type (%s).' % largeImagePath)
if ext.lower() in ('.jpg', '.jpeg', '.jpe', '.png'):
raise TileSourceException('File will not be opened by bioformats reader')
if not _startJavabridge(self._logger):
raise TileSourceException(
'File cannot be opened by bioformats reader because javabridge failed to start')
self._tileLock = threading.RLock()
try:
javabridge.attach()
try:
self._bioimage = bioformats.ImageReader(largeImagePath)
except AttributeError as exc:
self._logger.debug('File cannot be opened via Bioformats. (%r)' % exc)
raise TileSourceException('File cannot be opened via Bioformats. (%r)' % exc)
_openImages.append(self)
rdr = self._bioimage.rdr
# Bind additional functions not done by bioformats module.
# Functions are listed at https://downloads.openmicroscopy.org
# //bio-formats/5.1.5/api/loci/formats/IFormatReader.html
for (name, params, desc) in [
('getBitsPerPixel', '()I', 'Get the number of bits per pixel'),
('getEffectiveSizeC', '()I', 'effectiveC * Z * T = imageCount'),
('isNormalized', '()Z', 'Is float data normalized'),
('isMetadataComplete', '()Z', 'True if metadata is completely parsed'),
('getDomains', '()[Ljava/lang/String;', 'Get a list of domains'),
('getZCTCoords', '(I)[I', 'Gets the Z, C and T coordinates '
'(real sizes) corresponding to the given rasterized index value.'),
('getOptimalTileWidth', '()I', 'the optimal sub-image width '
'for use with openBytes'),
('getOptimalTileHeight', '()I', 'the optimal sub-image height '
'for use with openBytes'),
('getResolutionCount', '()I', 'The number of resolutions for '
'the current series'),
('setResolution', '(I)V', 'Set the resolution level'),
('getResolution', '()I', 'The current resolution level'),
('hasFlattenedResolutions', '()Z', 'True if resolutions have been flattened'),
('setFlattenedResolutions', '(Z)V', 'Set if resolution should be flattened'),
]:
setattr(rdr, name, types.MethodType(
javabridge.jutil.make_method(name, params, desc), rdr))
# rdr.setFlattenedResolutions(False)
self._metadata = {
'dimensionOrder': rdr.getDimensionOrder(),
'metadata': javabridge.jdictionary_to_string_dictionary(
rdr.getMetadata()),
'seriesMetadata': javabridge.jdictionary_to_string_dictionary(
rdr.getSeriesMetadata()),
'seriesCount': rdr.getSeriesCount(),
'imageCount': rdr.getImageCount(),
'rgbChannelCount': rdr.getRGBChannelCount(),
'sizeColorPlanes': rdr.getSizeC(),
'sizeT': rdr.getSizeT(),
'sizeZ': rdr.getSizeZ(),
'sizeX': rdr.getSizeX(),
'sizeY': rdr.getSizeY(),
'pixelType': rdr.getPixelType(),
'isLittleEndian': rdr.isLittleEndian(),
'isRGB': rdr.isRGB(),
'isInterleaved': rdr.isInterleaved(),
'isIndexed': rdr.isIndexed(),
'bitsPerPixel': rdr.getBitsPerPixel(),
'sizeC': rdr.getEffectiveSizeC(),
'normalized': rdr.isNormalized(),
'metadataComplete': rdr.isMetadataComplete(),
# 'domains': rdr.getDomains(),
'optimalTileWidth': rdr.getOptimalTileWidth(),
'optimalTileHeight': rdr.getOptimalTileHeight(),
'resolutionCount': rdr.getResolutionCount(),
'flattenedResolutions': rdr.hasFlattenedResolutions(),
}
self._checkSeries(rdr)
bmd = bioformats.metadatatools.MetadataRetrieve(self._bioimage.metadata)
try:
self._metadata['channelNames'] = [
bmd.getChannelName(0, c) or bmd.getChannelID(0, c)
for c in range(self._metadata['sizeColorPlanes'])]
except Exception:
self._metadata['channelNames'] = []
for key in ['sizeXY', 'sizeC', 'sizeZ', 'sizeT']:
if not isinstance(self._metadata[key], int) or self._metadata[key] < 1:
self._metadata[key] = 1
self.sizeX = self._metadata['sizeX']
self.sizeY = self._metadata['sizeY']
self._computeTiles()
self._computeLevels()
self._computeMagnification()
except javabridge.JavaException as exc:
es = javabridge.to_string(exc.throwable)
self._logger.debug('File cannot be opened via Bioformats. (%s)' % es)
raise TileSourceException('File cannot be opened via Bioformats. (%s)' % es)
except AttributeError:
self._logger.exception('The bioformats reader threw an unhandled exception.')
raise TileSourceException('The bioformats reader threw an unhandled exception.')
finally:
if javabridge.get_env():
javabridge.detach()
if self.levels < 1:
raise TileSourceException(
'OpenSlide image must have at least one level.')
if self.sizeX <= 0 or self.sizeY <= 0:
raise TileSourceException('Bioformats tile size is invalid.')
def __init__(self, df, penaltydiscount = 4, depth = 3, faithfulness = True, verbose = False, java_max_heap_size = None):
tetrad_libdir = os.path.join(os.path.dirname(__file__), 'lib')
# print 'tetrad_libdir: %s' % tetrad_libdir
for l in glob.glob(tetrad_libdir + os.sep + "*.jar"):
javabridge.JARS.append(str(l))
javabridge.start_vm(run_headless=True, max_heap_size = java_max_heap_size)
javabridge.attach()
tetradData = None
if(len(df.index)*df.columns.size <= 1500):
node_list = javabridge.JWrapper(javabridge.make_instance("java/util/ArrayList", "()V"))
for col in df.columns:
nodname = javabridge.make_instance("java/lang/String", "(Ljava/lang/String;)V",col)
nodi = javabridge.make_instance("edu/cmu/tetrad/graph/GraphNode", "(Ljava/lang/String;)V",nodname)
node_list.add(nodi)
tetradMatrix = javabridge.JWrapper(javabridge.make_instance("edu/cmu/tetrad/util/TetradMatrix","(II)V",
len(df.index),df.columns.size))
for row in df.index:
for col in range(0,df.columns.size):
tetradMatrix.set(row,col,df.ix[row][col])
tetradData = javabridge.static_call("edu/cmu/tetrad/data/ColtDataSet","makeContinuousData",
"(Ljava/util/List;Ledu/cmu/tetrad/util/TetradMatrix;)Ledu/cmu/tetrad/data/DataSet;",
node_list,tetradMatrix)
else:
#Generate random name
temp_data_file = ''.join(random.choice(string.lowercase) for i in range(10)) + '.csv'
temp_data_path = os.path.join(tempfile.gettempdir(), temp_data_file)
df.to_csv(temp_data_path, sep = "\t")
f = javabridge.make_instance("java/io/File", "(Ljava/lang/String;)V",temp_data_path)
excludeVar = javabridge.JWrapper(javabridge.make_instance("java/util/HashSet","()V"))
excludeVar.add("MULT")
tetradData = javabridge.static_call("edu/cmu/tetrad/data/BigDataSetUtility","readInContinuousData","(Ljava/io/File;CLjava/util/Set;)Ledu/cmu/tetrad/data/DataSet;",f,"\t",excludeVar)
os.remove(temp_data_path)
fgs = javabridge.make_instance("edu/cmu/tetrad/search/Fgs","(Ledu/cmu/tetrad/data/DataSet;)V",tetradData)
fgs = javabridge.JWrapper(fgs)
fgs.setPenaltyDiscount(penaltydiscount)# set to 2 if variable# <= 50 otherwise set it to 4
fgs.setDepth(depth)#-1
fgs.setNumPatternsToStore(0)
fgs.setFaithfulnessAssumed(faithfulness)
fgs.setVerbose(verbose)
tetradGraph = fgs.search()
graph = pydot.Dot(graph_type='digraph')
n = tetradGraph.getNodeNames().toString()
n = n[1:len(n)-1]
n = n.split(",")
nodes = []
for i in range(0,len(n)):
node = n[i]
n[i] = node.strip()
nodes.append(pydot.Node(n[i]))
graph.add_node(nodes[i])
self.nodes = n
e = tetradGraph.getEdges().toString()
e = e[1:len(e)-1]
e = e.split(",")
for i in range(0,len(e)):
e[i] = e[i].strip()
token = e[i].split(" ")
if(len(token) == 3):
src = token[0]
arc = token[1]
dst = token[2]
if(pycausal.isNodeExisting(n,src) and pycausal.isNodeExisting(n,dst)):
edge = pydot.Edge(nodes[n.index(src)],nodes[n.index(dst)])
if(arc == "---"):
edge.set_arrowhead("none")
graph.add_edge(edge)
self.edges = e
javabridge.detach()
javabridge.kill_vm()
self.graph = graph
def kill_vm():
javabridge.detach()
javabridge.kill_vm()
def tearDown(self):
J.detach()
def __init__(self,
df,
penaltydiscount=4,
depth=3,
faithfulness=True,
verbose=False,
java_max_heap_size=None):
tetrad_libdir = os.path.join(os.path.dirname(__file__), 'lib')
# print 'tetrad_libdir: %s' % tetrad_libdir
for l in glob.glob(tetrad_libdir + os.sep + "*.jar"):
javabridge.JARS.append(str(l))
javabridge.start_vm(run_headless=True,
max_heap_size=java_max_heap_size)
javabridge.attach()
tetradData = None
if (len(df.index) * df.columns.size <= 1500):
node_list = javabridge.JWrapper(
javabridge.make_instance("java/util/ArrayList", "()V"))
for col in df.columns:
nodname = javabridge.make_instance("java/lang/String",
"(Ljava/lang/String;)V",
col)
nodi = javabridge.make_instance(
"edu/cmu/tetrad/graph/GraphNode", "(Ljava/lang/String;)V",
nodname)
node_list.add(nodi)
tetradMatrix = javabridge.JWrapper(
javabridge.make_instance("edu/cmu/tetrad/util/TetradMatrix",
"(II)V", len(df.index),
df.columns.size))
for row in df.index:
for col in range(0, df.columns.size):
tetradMatrix.set(row, col, df.ix[row][col])
tetradData = javabridge.static_call(
"edu/cmu/tetrad/data/ColtDataSet", "makeContinuousData",
"(Ljava/util/List;Ledu/cmu/tetrad/util/TetradMatrix;)Ledu/cmu/tetrad/data/DataSet;",
node_list, tetradMatrix)
else:
#Generate random name
temp_data_file = ''.join(
random.choice(string.lowercase) for i in range(10)) + '.csv'
temp_data_path = os.path.join(tempfile.gettempdir(),
temp_data_file)
df.to_csv(temp_data_path, sep="\t")
f = javabridge.make_instance("java/io/File",
"(Ljava/lang/String;)V",
temp_data_path)
excludeVar = javabridge.JWrapper(
javabridge.make_instance("java/util/HashSet", "()V"))
excludeVar.add("MULT")
tetradData = javabridge.static_call(
"edu/cmu/tetrad/data/BigDataSetUtility",
"readInContinuousData",
"(Ljava/io/File;CLjava/util/Set;)Ledu/cmu/tetrad/data/DataSet;",
f, "\t", excludeVar)
os.remove(temp_data_path)
fgs = javabridge.make_instance("edu/cmu/tetrad/search/Fgs",
"(Ledu/cmu/tetrad/data/DataSet;)V",
tetradData)
fgs = javabridge.JWrapper(fgs)
fgs.setPenaltyDiscount(
penaltydiscount
) # set to 2 if variable# <= 50 otherwise set it to 4
fgs.setDepth(depth) #-1
fgs.setNumPatternsToStore(0)
fgs.setFaithfulnessAssumed(faithfulness)
fgs.setVerbose(verbose)
tetradGraph = fgs.search()
graph = pydot.Dot(graph_type='digraph')
n = tetradGraph.getNodeNames().toString()
n = n[1:len(n) - 1]
n = n.split(",")
nodes = []
for i in range(0, len(n)):
node = n[i]
n[i] = node.strip()
nodes.append(pydot.Node(n[i]))
graph.add_node(nodes[i])
self.nodes = n
e = tetradGraph.getEdges().toString()
e = e[1:len(e) - 1]
e = e.split(",")
for i in range(0, len(e)):
e[i] = e[i].strip()
token = e[i].split(" ")
if (len(token) == 3):
src = token[0]
arc = token[1]
dst = token[2]
if (pycausal.isNodeExisting(n, src)
and pycausal.isNodeExisting(n, dst)):
edge = pydot.Edge(nodes[n.index(src)], nodes[n.index(dst)])
if (arc == "---"):
edge.set_arrowhead("none")
graph.add_edge(edge)
self.edges = e
javabridge.detach()
javabridge.kill_vm()
self.graph = graph
