def _getAssociatedImage(self, imageKey): """ Get an associated image in PIL format. :param imageKey: the key of the associated image. :return: the image in PIL format or None. """ info = self._metadata['seriesAssociatedImages'].get(imageKey) if info is None: return series = info['seriesNum'] with self._tileLock: try: javabridge.attach() image = self._bioimage.read( series=series, rescale=False, # return internal data types XYWH=(0, 0, info['sizeX'], info['sizeY'])) except javabridge.JavaException as exc: es = javabridge.to_string(exc.throwable) raise TileSourceException('Failed to get Bioformat series (%s, %r).' % (es, ( series, info['sizeX'], info['sizeY']))) finally: if javabridge.get_env(): javabridge.detach() return large_image.tilesource.base._imageToPIL(image)
def write_tile(br, bw, X, Y, Z, C): """Read and write one tile of a multichannel image This function is intended to be run inside of a thread. Since data is sent to the JVM and the GIL is released, reading images can be performed in parallel. Writing still requires a lock. Args: br ([bfio.BioReader]): BioReader to be read from bw ([bfio.BioWriter]): BioWriter to be written to X ([list]): Range of x-values to read/write Y ([list]): Range of y-values to grab/write C ([list]): Channel to write """ # Connect to the JVM jutil.attach() # Read the image image = br.read_image(X=X, Y=Y, Z=Z) # Lock the thread and write the image with lock: bw.write_image(image, X=[X[0]], Y=[Y[0]], Z=[Z[0]], C=C) # Detach from the JVM jutil.detach()
def exit_thread(self): from bioformats.formatreader import clear_image_reader_cache self.notify_socket.close() clear_image_reader_cache() J.detach() if self.with_stop_run_loop: stop_run_loop()
def tearDownModule(): try: os.environ['PYJNIUS_ACTIVATE'] except KeyError: import javabridge javabridge.detach() javabridge.kill_vm()
def validation_queue_handler(): from javabridge import attach, detach attach() try: while mv_constants.validation_queue_keep_running: request = mv_constants.validation_queue.get() if ( not isinstance(request, ValidationRequestController) or request.cancelled ): continue start = time.perf_counter() try: validate_module( request.pipeline, request.module_num, request.callback, ) except: pass # Make sure this thread utilizes less than 1/2 of GIL clock wait_for = max(0.25, time.perf_counter() - start) time.sleep(wait_for) finally: detach() logging.info("Exiting the pipeline validation thread")
def tearDownModule(): try: os.environ["PYJNIUS_ACTIVATE"] except KeyError: import javabridge javabridge.detach() javabridge.kill_vm()
def __call__(self, id, filename, uri, **kwargs): """ Extract metadata from a Datafile using the get_meta function and save the outputs as DatafileParameters. This function differs from process_meta in that it generates an output directory in the metadata store and passes it to the metadata processing func so that outputs e.g., preview images or metadata files) can be saved. param id: Datafile ID type id: integer param filename: Absolute path to a file for processing type filename: string param uri: Dataset URI type uri: string param kwargs: Extra arguments type kwargs: object return Extracted metadata rtype dict """ _, extension = os.path.splitext(filename) if extension[1:] not in bioformats.READABLE_FORMATS: return None logger.info("Applying Bioformats filter to {}...".format(filename)) # Need to start a JVM in each thread check_and_start_jvm() try: javabridge.attach() shush_logger() thumb_rel_path, thumb_abs_path = get_thumbnail_paths( id, filename, uri) if not os.path.exists(os.path.dirname(thumb_abs_path)): os.makedirs(os.path.dirname(thumb_abs_path)) rsp = get_meta(filename, os.path.dirname(thumb_abs_path), **kwargs) if rsp is not None: metadata = [] for i in rsp: metadata.append(self.filter_metadata(i)) return metadata[0] except Exception as e: logger.error(str(e)) logger.debug(traceback.format_exc()) finally: javabridge.detach() javabridge.kill_vm() return None
def __del__(self): if getattr(self, '_bioimage', None) is not None: try: javabridge.attach() self._bioimage.close() _openImages.remove(self) finally: if javabridge.get_env(): javabridge.detach()
def run(self): javabridge.attach() try: self.socket = self.context.socket(zmq.REP) self.socket.bind(self.address) logger.info("Binding Knime bridge server to %s" % self.address) poller = zmq.Poller() poller.register(self.socket, flags=zmq.POLLIN) if self.notify_addr is not None: self.notify_socket = self.context.socket(zmq.SUB) self.notify_socket.setsockopt(zmq.SUBSCRIBE, "") self.notify_socket.connect(self.notify_addr) poller.register(self.notify_socket, flags=zmq.POLLIN) else: self.notify_socket = None start_socket = self.context.socket(zmq.PAIR) start_socket.connect(self.start_addr) start_socket.send("OK") start_socket.close() try: while True: for socket, event in poller.poll(1): if socket == self.notify_socket: msg = self.notify_socket.recv_string() if msg == self.stop_msg: break elif socket == self.socket: msg = self.socket.recv_multipart(copy=False) session_id_frame = msg.pop(0) session_id = str(session_id_frame.bytes) # should be None wrapper = msg.pop(0) message_type = msg.pop(0).bytes if message_type not in self.dispatch: self.raise_cellprofiler_exception( session_id, "Unhandled message type: %s" % message_type, ) else: try: self.dispatch[message_type](session_id, message_type, msg) except Exception as e: logger.warn(e.message, exc_info=1) self.raise_cellprofiler_exception( session_id, e.message) else: continue break finally: if self.notify_socket: self.notify_socket.close() self.socket.close() finally: javabridge.detach()
def run(self): javabridge.attach() try: self.socket = self.context.socket(zmq.REP) self.socket.bind(self.address) logger.info("Binding Knime bridge server to %s" % self.address) poller = zmq.Poller() poller.register(self.socket, flags=zmq.POLLIN) if self.notify_addr is not None: self.notify_socket = self.context.socket(zmq.SUB) self.notify_socket.setsockopt(zmq.SUBSCRIBE, "") self.notify_socket.connect(self.notify_addr) poller.register(self.notify_socket, flags=zmq.POLLIN) else: self.notify_socket = None start_socket = self.context.socket(zmq.PAIR) start_socket.connect(self.start_addr) start_socket.send("OK") start_socket.close() try: while True: for socket, event in poller.poll(1): if socket == self.notify_socket: msg = self.notify_socket.recv_string() if msg == self.stop_msg: break elif socket == self.socket: msg = self.socket.recv_multipart(copy=False) session_id_frame = msg.pop(0) session_id = str(session_id_frame.bytes) # should be None wrapper = msg.pop(0) message_type = msg.pop(0).bytes if message_type not in self.dispatch: self.raise_cellprofiler_exception( session_id, "Unhandled message type: %s" % message_type) else: try: self.dispatch[message_type]( session_id, message_type, msg) except Exception, e: logger.warn(e.message, exc_info=1) self.raise_cellprofiler_exception( session_id, e.message) else: continue break finally: if self.notify_socket: self.notify_socket.close() self.socket.close() finally: javabridge.detach()
def run(self): if p.force_bioformats: logging.debug("Starting javabridge") import bioformats javabridge.start_vm(class_path=bioformats.JARS, run_headless=True) javabridge.attach() try: while 1: self.tile_collection.cv.acquire() # If there are no objects in the queue then wait while not self.tile_collection.loadq: self.tile_collection.cv.wait() if self._want_abort: self.tile_collection.cv.release() db = DBConnect() db.CloseConnection() logging.info('%s aborted' % self.getName()) return data = heappop(self.tile_collection.loadq) obKey = data[1] display_whole_image = data[ 2] #display whole image instead of object image self.tile_collection.cv.release() # wait until loading has completed before continuing with self.tile_collection.load_lock: # Make sure tile hasn't been deleted outside this thread if not self.tile_collection.tileData.get(obKey, None): continue # Get the tile new_data = imagetools.FetchTile( obKey, display_whole_image=display_whole_image) if new_data is None: #if fetching fails, leave the tile blank continue tile_data = self.tile_collection.tileData.get(obKey, None) # Make sure tile hasn't been deleted outside this thread if tile_data is not None: # copy each channel for i in range(len(tile_data)): tile_data[i] = new_data[i] for window in self.notify_window: wx.PostEvent(window, TileUpdatedEvent(obKey)) finally: if javabridge.get_env() is not None: javabridge.detach()
def engine(q_request, q_response): logger.info("Starting script engine thread") J.attach() while True: try: msg = J.run_script( """importPackage(Packages.org.scijava.plugins.scripting.cpython); q_request.take();""", dict(q_request=q_request)) if logger.level <= logging.INFO: logger.info("Received engine request: %s", J.to_string(msg)) payload = J.get_collection_wrapper( J.run_script("msg.payload", dict(msg=msg))) if J.run_script( """importPackage(Packages.org.scijava.plugins.scripting.cpython); msg.command==CPythonScriptEngine.EngineCommands.EXECUTE; """, dict(msg=msg)): response = do_execute(payload) elif J.run_script( """importPackage(Packages.org.scijava.plugins.scripting.cpython); msg.command==CPythonScriptEngine.EngineCommands.EVALUATE; """, dict(msg=msg)): response = do_evaluate(payload) elif J.run_script( """importPackage(Packages.org.scijava.plugins.scripting.cpython); msg.command==CPythonScriptEngine.EngineCommands.CLOSE_ENGINE; """, dict(msg=msg)): logger.info("Exiting script engine thread after close request") break else: logger.warn( "Received unknown command: %s" % J.run_script("msg.command.toString()", dict(msg=msg))) response = J.run_script( """importPackage(Packages.org.scijava.plugins.scripting.cpython); var exception = new java.lang.RuntimeException( java.lang.String.format('Unknown command: %s', msg.command.toString())); var payload = new java.util.ArrayList(); payload.add(exception); new CPythonScriptEngine.Message( CPythonScriptEngine.EngineCommands.EXCEPTION, payload); """, dict(msg=msg)) J.run_script("q_response.put(response);", dict(q_response=q_response, response=response)) except: # To do: how to handle failure, probably from .take() # Guessing that someone has managed to interrupt our thread logger.warn("Exiting script engine thread", exc_info=True) J.detach()
def _monitor_thread(): main_thread = threading.main_thread() main_thread.join() if len(_openImages): try: javabridge.attach() while len(_openImages): source = _openImages.pop() try: source._bioimage.close() except Exception: pass source._bioimage = None finally: if javabridge.get_env(): javabridge.detach() _stopJavabridge()
def run(self): javabridge.attach() try: while 1: self.tile_collection.cv.acquire() # If there are no objects in the queue then wait while not self.tile_collection.loadq: self.tile_collection.cv.wait() if self._want_abort: self.tile_collection.cv.release() logging.info('%s aborted' % self.getName()) return data = heappop(self.tile_collection.loadq) obKey = data[1] display_whole_image = data[ 2] #display whole image instead of object image self.tile_collection.cv.release() # wait until loading has completed before continuing with self.tile_collection.load_lock: # Make sure tile hasn't been deleted outside this thread if not self.tile_collection.tileData.get(obKey, None): continue # Get the tile new_data = imagetools.FetchTile( obKey, display_whole_image=display_whole_image) if new_data is None: #if fetching fails, leave the tile blank continue tile_data = self.tile_collection.tileData.get(obKey, None) # Make sure tile hasn't been deleted outside this thread if tile_data is not None: # copy each channel for i in range(len(tile_data)): tile_data[i] = new_data[i] wx.PostEvent(self.notify_window, TileUpdatedEvent(obKey)) finally: javabridge.detach()
def run(self): javabridge.attach() try: while 1: self.tile_collection.cv.acquire() # If there are no objects in the queue then wait while not self.tile_collection.loadq: self.tile_collection.cv.wait() if self._want_abort: self.tile_collection.cv.release() logging.info("%s aborted" % self.getName()) return data = heappop(self.tile_collection.loadq) obKey = data[1] display_whole_image = data[2] # display whole image instead of object image self.tile_collection.cv.release() # wait until loading has completed before continuing with self.tile_collection.load_lock: # Make sure tile hasn't been deleted outside this thread if not self.tile_collection.tileData.get(obKey, None): continue # Get the tile new_data = imagetools.FetchTile(obKey, display_whole_image=display_whole_image) if new_data is None: # if fetching fails, leave the tile blank continue tile_data = self.tile_collection.tileData.get(obKey, None) # Make sure tile hasn't been deleted outside this thread if tile_data is not None: # copy each channel for i in range(len(tile_data)): tile_data[i] = new_data[i] wx.PostEvent(self.notify_window, TileUpdatedEvent(obKey)) finally: javabridge.detach()
def fn(): from bioformats import load_image_url import javabridge from scipy.io.matlab.mio import loadmat from cellprofiler.modules.loadimages import url2pathname javabridge.attach() with self.image_dict_lock: generation = self.image_dict_generation for k, v in well.iteritems(): sd = {} with self.image_dict_lock: if self.image_dict_generation > generation: return self.image_dict[k] = sd for c, fd in enumerate(v): if PlateData.D_CHANNEL in fd: channel = fd[PlateData.D_CHANNEL] else: channel = str(c + 1) url = fd[PlateData.D_FILENAME] try: if url.lower().endswith(".mat"): img = loadmat( url2pathname(url), struct_as_record=True)["Image"] else: img = load_image_url(url) with self.image_dict_lock: if self.image_dict_generation > generation: return sd[channel] = img except: traceback.print_exc() pass wx.CallAfter(self.update_figure) javabridge.static_call( "java/lang/System", "gc", "()V") javabridge.detach()
def fn(): from bioformats import load_image_url import javabridge from scipy.io.matlab.mio import loadmat from cellprofiler.modules.loadimages import url2pathname javabridge.attach() with self.image_dict_lock: generation = self.image_dict_generation for k, v in well.iteritems(): sd = {} with self.image_dict_lock: if self.image_dict_generation > generation: return self.image_dict[k] = sd for c, fd in enumerate(v): if PlateData.D_CHANNEL in fd: channel = fd[PlateData.D_CHANNEL] else: channel = str(c+1) url = fd[PlateData.D_FILENAME] try: if url.lower().endswith(".mat"): img = loadmat( url2pathname(url), struct_as_record=True)["Image"] else: img = load_image_url(url) with self.image_dict_lock: if self.image_dict_generation > generation: return sd[channel] = img except: traceback.print_exc() pass wx.CallAfter(self.update_figure) javabridge.static_call( "java/lang/System", "gc", "()V") javabridge.detach()
def engine_requester(): J.attach() while True: try: msg = J.run_script( """importPackage(Packages.org.scijava.plugins.scripting.cpython); CPythonScriptEngine.engineRequestQueue.take();""") if logger.level <= logging.INFO: logger.info("Received engine request: %s", J.to_string(msg)) payload = J.get_collection_wrapper( J.run_script("msg.payload", dict(msg=msg))) if J.run_script( """importPackage(Packages.org.scijava.plugins.scripting.cpython); msg.command==CPythonScriptEngine.EngineCommands.NEW_ENGINE; """, dict(msg=msg)): do_new_engine(payload) elif J.run_script( """importPackage(Packages.org.scijava.plugins.scripting.cpython); msg.command==CPythonScriptEngine.EngineCommands.CLOSE_SERVICE; """, dict(msg=msg)): logger.info("Exiting script service thread in response to " "termination request") break else: J.run_script( """importPackage(Packages.org.scijava.plugins.scripting.cpython); var exception = new java.lang.RuntimeException( java.lang.String.format('Unknown command: %s', msg.command.toString())); var payload = new java.util.ArrayList(); payload.add(exception); var response = new CPythonScriptEngine.Message( CPythonScriptEngine.EngineCommands.EXCEPTION, payload); CPythonScriptEngine.engineResponseQueue.put(response); """) except: # To do: how to handle failure, probably from .take() # Guessing that someone has managed to interrupt our thread logger.warn("Exiting script service thread", exc_info=True) J.detach()
def detach(self): javabridge.detach()
def __init__(self, df, penaltydiscount = 4, depth = 3, faithfulness = True, numofthreads = 2, verbose = False, java_max_heap_size = None): tetrad_libdir = os.path.join(os.path.dirname(__file__), 'lib') for l in glob.glob(tetrad_libdir + os.sep + "*.jar"): javabridge.JARS.append(str(l)) javabridge.start_vm(run_headless=True, max_heap_size = java_max_heap_size) javabridge.attach() tetradData = None if(len(df.index)*df.columns.size <= 1500): node_list = javabridge.JClassWrapper('java.util.ArrayList')() for col in df.columns: nodi = javabridge.JClassWrapper('edu.cmu.tetrad.data.ContinuousVariable')(col) node_list.add(nodi) dataBox = javabridge.JClassWrapper('edu.cmu.tetrad.data.DoubleDataBox')(len(df.index),df.columns.size) for row in df.index: for col in range(0,df.columns.size): value = javabridge.JClassWrapper('java.lang.Double')(df.ix[row][col]) dataBox.set(row,col,value) tetradData = javabridge.JClassWrapper('edu.cmu.tetrad.data.BoxDataSet')(dataBox, node_list) else: #Generate random name temp_data_file = ''.join(random.choice(string.lowercase) for i in range(10)) + '.csv' temp_data_path = os.path.join(tempfile.gettempdir(), temp_data_file) df.to_csv(temp_data_path, sep = '\t', index = False) excludeVar = javabridge.JClassWrapper('java.util.HashSet')() excludeVar.add('MULT') # Read in method 1 f = javabridge.JClassWrapper('java.nio.file.Paths').get(temp_data_path) dataReader = javabridge.JClassWrapper('edu.cmu.tetrad.io.TabularContinuousDataReader')(f,'\t') tetradData = dataReader.readInData(excludeVar) # Read in method 2 -- Depreciated # f = javabridge.JClassWrapper('java.io.File')(temp_data_path) # tetradData = javabridge.JClassWrapper('edu.cmu.tetrad.data.BigDataSetUtility').readInContinuousData(f,'\t',excludeVar) os.remove(temp_data_path) tetradData = javabridge.JClassWrapper('edu.cmu.tetrad.data.CovarianceMatrixOnTheFly')(tetradData) score = javabridge.JClassWrapper('edu.cmu.tetrad.search.SemBicScore')(tetradData) score.setPenaltyDiscount(penaltydiscount) # set to 2 if variable# <= 50 otherwise set it to 4 fgs = javabridge.JClassWrapper('edu.cmu.tetrad.search.Fgs')(score) fgs.setDepth(depth)#-1 fgs.setNumPatternsToStore(0) fgs.setFaithfulnessAssumed(faithfulness) fgs.setParallelism(numofthreads) fgs.setVerbose(verbose) tetradGraph = fgs.search() graph = pydot.Dot(graph_type='digraph') n = tetradGraph.getNodeNames().toString() n = n[1:len(n)-1] n = n.split(",") nodes = [] for i in range(0,len(n)): node = n[i] n[i] = node.strip() nodes.append(pydot.Node(n[i])) graph.add_node(nodes[i]) self.nodes = n e = tetradGraph.getEdges().toString() e = e[1:len(e)-1] e = e.split(",") for i in range(0,len(e)): e[i] = e[i].strip() token = e[i].split(" ") if(len(token) == 3): src = token[0] arc = token[1] dst = token[2] if(pycausal.isNodeExisting(n,src) and pycausal.isNodeExisting(n,dst)): edge = pydot.Edge(nodes[n.index(src)],nodes[n.index(dst)]) if(arc == "---"): edge.set_arrowhead("none") graph.add_edge(edge) self.edges = e javabridge.detach() javabridge.kill_vm() self.graph = graph
def tearDownClass(cls): J.detach()
def tearDown(self): J.detach()
def run(self): javabridge.attach() try: main(sys.argv) finally: javabridge.detach()
def tearDown(self): javabridge.detach()
def run(my_integer=my_integer): env = javabridge.attach() self.assertEqual(my_integer.intValue(), my_value) del my_integer javabridge.detach()
def stop_vm(self): javabridge.detach() javabridge.kill_vm()
def get_user_loader(): '''The class loader used to load user plugins''' return J.static_call("ij/IJ", "getClassLoader", "()Ljava/lang/ClassLoader;") def get_plugin(classname): '''Return an instance of the named plugin''' if classname.startswith("ij."): cls = J.class_for_name(classname) else: cls = J.class_for_name(classname, get_user_loader()) cls = J.get_class_wrapper(cls, True) constructor = J.get_constructor_wrapper(cls.getConstructor(None)) return constructor.newInstance(None) if __name__=="__main__": import sys J.attach() try: commands = get_commands() print "Commands: " for command in commands: print "\t" + command if len(sys.argv) == 2: execute_command(sys.argv[1]) elif len(sys.argv) > 2: execute_command(sys.argv[1], sys.argv[2]) finally: J.detach() J.kill_vm()
def getTile(self, x, y, z, pilImageAllowed=False, numpyAllowed=False, **kwargs): self._xyzInRange(x, y, z) ft = fc = fz = 0 fseries = self._metadata['frameSeries'][0] if kwargs.get('frame') is not None: frame = int(kwargs.get('frame')) fc = frame % self._metadata['sizeC'] fz = (frame // self._metadata['sizeC']) % self._metadata['sizeZ'] ft = (frame // self._metadata['sizeC'] // self._metadata['sizeZ']) % self._metadata['sizeT'] fxy = (frame // self._metadata['sizeC'] // self._metadata['sizeZ'] // self._metadata['sizeT']) if frame < 0 or fxy > self._metadata['sizeXY']: raise TileSourceException('Frame does not exist') fseries = self._metadata['frameSeries'][fxy] seriesLevel = self.levels - 1 - z scale = 1 while seriesLevel >= len(fseries['series']): seriesLevel -= 1 scale *= 2 offsetx = x * self.tileWidth * scale offsety = y * self.tileHeight * scale width = min(self.tileWidth * scale, self.sizeX // 2 ** seriesLevel - offsetx) height = min(self.tileHeight * scale, self.sizeY // 2 ** seriesLevel - offsety) sizeXAtScale = fseries['sizeX'] // (2 ** seriesLevel) sizeYAtScale = fseries['sizeY'] // (2 ** seriesLevel) finalWidth = width // scale finalHeight = height // scale width = min(width, sizeXAtScale - offsetx) height = min(height, sizeYAtScale - offsety) with self._tileLock: try: javabridge.attach() if width > 0 and height > 0: tile = self._bioimage.read( c=fc, z=fz, t=ft, series=fseries['series'][seriesLevel], rescale=False, # return internal data types XYWH=(offsetx, offsety, width, height)) else: # We need the same dtype, so read 1x1 at 0x0 tile = self._bioimage.read( c=fc, z=fz, t=ft, series=fseries['series'][seriesLevel], rescale=False, # return internal data types XYWH=(0, 0, 1, 1)) tile = numpy.zeros(tuple([0, 0] + list(tile.shape[2:])), dtype=tile.dtype) format = TILE_FORMAT_NUMPY except javabridge.JavaException as exc: es = javabridge.to_string(exc.throwable) raise TileSourceException('Failed to get Bioformat region (%s, %r).' % (es, ( fc, fz, ft, fseries, self.sizeX, self.sizeY, offsetx, offsety, width, height))) finally: if javabridge.get_env(): javabridge.detach() if scale > 1: tile = tile[::scale, ::scale] if tile.shape[:2] != (finalHeight, finalWidth): fillValue = 0 if tile.dtype == numpy.uint16: fillValue = 65535 elif tile.dtype == numpy.uint8: fillValue = 255 elif tile.dtype.kind == 'f': fillValue = 1 retile = numpy.full( tuple([finalHeight, finalWidth] + list(tile.shape[2:])), fillValue, dtype=tile.dtype) retile[0:min(tile.shape[0], finalHeight), 0:min(tile.shape[1], finalWidth)] = tile[ 0:min(tile.shape[0], finalHeight), 0:min(tile.shape[1], finalWidth)] tile = retile return self._outputTile(tile, format, x, y, z, pilImageAllowed, numpyAllowed, **kwargs)
def __init__(self, df, structurePrior = 1.0, samplePrior = 1.0, depth = 3, faithfulness = True, numofthreads = 2, verbose = False, java_max_heap_size = None): tetrad_libdir = os.path.join(os.path.dirname(__file__), 'lib') for l in glob.glob(tetrad_libdir + os.sep + "*.jar"): javabridge.JARS.append(str(l)) javabridge.start_vm(run_headless=True, max_heap_size = java_max_heap_size) javabridge.attach() score = None # if(len(df.index)*df.columns.size <= 1500): dataBox = javabridge.JClassWrapper('edu.cmu.tetrad.data.VerticalIntDataBox')(len(df.index),df.columns.size) node_list = javabridge.JClassWrapper('java.util.ArrayList')() col_no = 0 for col in df.columns: cat_array = sorted(set(df[col])) cat_list = javabridge.JClassWrapper('java.util.ArrayList')() for cat in cat_array: catname = javabridge.JClassWrapper('java.lang.String')(cat) cat_list.add(catname) nodname = javabridge.JClassWrapper('java.lang.String')(col) nodi = javabridge.JClassWrapper('edu.cmu.tetrad.data.DiscreteVariable')(nodname,cat_list) node_list.add(nodi) for row in df.index: value = javabridge.JClassWrapper('java.lang.Integer')(cat_array.index(df.ix[row][col_no])) dataBox.set(row,col_no,value) col_no = col_no + 1 tetradData = javabridge.JClassWrapper('edu.cmu.tetrad.data.BoxDataSet')(dataBox, node_list) # else: # Generate random name # temp_data_file = ''.join(random.choice(string.lowercase) for i in range(10)) + '.csv' # temp_data_path = os.path.join(tempfile.gettempdir(), temp_data_file) # df.to_csv(temp_data_path, sep = "\t", index = False) # excludeVar = javabridge.JWrapper(javabridge.make_instance("java/util/HashSet","()V")) # excludeVar.add("MULT") # Read in method 1 # f = javabridge.static_call("java/nio/file/Paths","get","(Ljava/lang/String;)V",temp_data_path) # dataReader = javabridge.make_instance("edu/cmu/tetrad/io/TabularContinuousDataReader","(Ljava/nio/file/Path;C)V",f,"\t") # dataReader = javabridge.JWrapper(dataReader) # tetradData = dataReader.readInData(excludeVar) # Read in method 2 -- Depreciated # f = javabridge.make_instance("java/io/File", "(Ljava/lang/String;)V",temp_data_path) # tetradData = javabridge.static_call("edu/cmu/tetrad/data/BigDataSetUtility", "readInDiscreteData", "(Ljava/io/File;CLjava/util/Set;)Ledu/cmu/tetrad/data/DataSet;",f,"\t",excludeVar) # os.remove(temp_data_path) score = javabridge.JClassWrapper('edu.cmu.tetrad.search.BDeuScore')(tetradData) score.setStructurePrior(structurePrior) score.setSamplePrior(samplePrior) fgs = javabridge.JClassWrapper('edu.cmu.tetrad.search.Fgs')(score) fgs.setDepth(depth) fgs.setNumPatternsToStore(0) fgs.setFaithfulnessAssumed(faithfulness) fgs.setParallelism(numofthreads) fgs.setVerbose(verbose) tetradGraph = fgs.search() graph = pydot.Dot(graph_type='digraph') n = tetradGraph.getNodeNames().toString() n = n[1:len(n)-1] n = n.split(",") nodes = [] for i in range(0,len(n)): node = n[i] n[i] = node.strip() nodes.append(pydot.Node(n[i])) graph.add_node(nodes[i]) self.nodes = n e = tetradGraph.getEdges().toString() e = e[1:len(e)-1] e = e.split(",") for i in range(0,len(e)): e[i] = e[i].strip() token = e[i].split(" ") if(len(token) == 3): src = token[0] arc = token[1] dst = token[2] if(pycausal.isNodeExisting(n,src) and pycausal.isNodeExisting(n,dst)): edge = pydot.Edge(nodes[n.index(src)],nodes[n.index(dst)]) if(arc == "---"): edge.set_arrowhead("none") graph.add_edge(edge) self.edges = e javabridge.detach() javabridge.kill_vm() self.graph = graph
def tearDownClass(self): J.detach()
def process_meta(df_id, schema_name, overwrite=False, **kwargs): """Extract metadata from a Datafile using the get_meta function and save the outputs as DatafileParameters. Parameters ---------- df_id: int ID of Datafile instance to process. schema_name: str Names of schema which describes ParameterNames add: boolean (default: False) Specifies whether or not to add to an existing Parameterset for this Datafile rather that overwriting or exiting. overwrite: boolean (default: False) Specifies whether to overwrite any exisiting parametersets for this datafile. Returns ------- None """ from .metadata import get_meta if acquire_datafile_lock(df_id): # Need to start a JVM in each thread check_and_start_jvm() try: javabridge.attach() log4j.basic_config() schema = Schema.objects.get(namespace__exact=schema_name) df = DataFile.objects.get(id=df_id) if DatafileParameterSet.objects\ .filter(schema=schema, datafile=df).exists(): if overwrite: psets = DatafileParameterSet.objects.get(schema=schema, datafile=df) logger.warning("Overwriting parametersets for %s", df.filename) for ps in psets: delete_old_parameterset(ps) else: logger.warning("Parametersets for %s already exist.", df.filename) return dfo = DataFileObject.objects.filter(datafile__id=df.id, verified=True).first() input_file_path = dfo.get_full_path() logger.debug("Processing file: %s", input_file_path) metadata_params = get_meta(input_file_path, **kwargs) if not metadata_params: logger.debug("No metadata to save") return for sm in metadata_params: ps = DatafileParameterSet(schema=schema, datafile=df) ps.save() logger.debug("Saving parameters for: %s", input_file_path) save_parameters(schema, ps, sm) except Exception as err: logger.exception(err) finally: release_datafile_lock(df_id) javabridge.detach()
def process_meta_file_output(df_id, schema_name, overwrite=False, **kwargs): """Extract metadata from a Datafile using the get_meta function and save the outputs as DatafileParameters. This function differs from process_meta in that it generates an output directory in the metadata store and passes it to the metadata processing func so that outputs (e.g., preview images or metadata files) can be saved. Parameters ---------- df_id: int ID of Datafile instance to process. schema_name: str Names of schema which describes ParameterNames add: Boolean (default: False) Specifies whether or not to add to an existing Parameterset for this Datafile rather that overwriting or exiting. overwrite: Boolean (default: False) Specifies whether to overwrite any exisiting parametersets for this datafile. Returns ------- None """ from .metadata import get_meta if acquire_datafile_lock(df_id): # Need to start a JVM in each thread check_and_start_jvm() try: javabridge.attach() log4j.basic_config() schema = Schema.objects.get(namespace__exact=schema_name) df = DataFile.objects.get(id=df_id) if DatafileParameterSet.objects\ .filter(schema=schema, datafile=df).exists(): if overwrite: psets = DatafileParameterSet.objects.get(schema=schema, datafile=df) logger.warning("Overwriting parametersets for %s", df.filename) for ps in psets: delete_old_parameterset(ps) else: logger.warning("Parametersets for %s already exist.", df.filename) return dfo = DataFileObject.objects.filter(datafile__id=df.id, verified=True).first() input_file_path = dfo.get_full_path() output_rel_path = os.path.join( os.path.dirname(urlparse.urlparse(dfo.uri).path), str(df.id)) output_path = os.path.join(settings.METADATA_STORE_PATH, output_rel_path) if not os.path.exists(output_path): os.makedirs(output_path) logger.debug("Processing file: %s" % input_file_path) metadata_params = get_meta(input_file_path, output_path, **kwargs) if not metadata_params: logger.debug("No metadata to save") return for sm in metadata_params: ps = DatafileParameterSet(schema=schema, datafile=df) ps.save() logger.debug("Saving parameters for: %s", input_file_path) save_parameters(schema, ps, sm) except Exception as err: logger.exception(err) finally: release_datafile_lock(df_id) javabridge.detach()
metadata_params = func(input_file_path, output_path, **kwargs) if not metadata_params: logger.debug("No metadata to save") return for sm in metadata_params: ps = DatafileParameterSet(schema=schema, datafile=df) ps.save() logger.debug("Saving parameters for: %s" % input_file_path) save_parameters(schema, ps, sm) except Exception, e: logger.debug(e) finally: release_datafile_lock(df.id) javabridge.detach() @task(name="mytardisbf.process_meta", ignore_result=True) def process_meta(func, df, schema_name, overwrite=False, **kwargs): """Extract metadata from a Datafile using a provided function and save the outputs as DatafileParameters. Parameters ---------- func: Function Function to extract metadata from a file. Function must have input_file_path as an argument e.g.: def meta_proc(input_file_path, **kwargs): ...
def exit_thread(self): from bioformats.formatreader import clear_image_reader_cache self.notify_socket.close() clear_image_reader_cache() J.detach()
print('big endian:', meta.getPixelsBigEndian(0, 0)) print('dim order:', meta.getPixelsDimensionOrder(0, 0)) print('pixel type:', meta.getPixelsPixelType(0, 0)) print('size x:', meta.getPixelsSizeX(0, 0)) print('size y:', meta.getPixelsSizeY(0, 0)) print('size c:', meta.getPixelsSizeC(0, 0)) print('size z:', meta.getPixelsSizeZ(0, 0)) print('size t:', meta.getPixelsSizeT(0, 0)) print('samples per pixel:', meta.getLogicalChannelSamplesPerPixel(0, 0)) writer.setMetadataRetrieve(meta) writer.setId(out_file) for image in images: if len(image.shape)==3 and image.shape[2] == 3: save_im = np.array([image[:,:,0], image[:,:,1], image[:,:,2]]).astype(np.uint8).flatten() else: save_im = image.astype(np.uint8).flatten() writer.saveBytes(env.make_byte_array(save_im), (image is images[-1])) writer.close() print('Done writing image :)') # import PIL.Image as Image # im = Image.open(out_file, 'r') # im.show() jutil.detach() app.MainLoop()
def run(my_integer = my_integer): env = javabridge.attach() self.assertEqual(my_integer.intValue(),my_value) del my_integer javabridge.detach()
def interface(self, start_signal, image_set_start=1, image_set_end=None, overwrite=True): '''Top-half thread for running an analysis. Sets up grouping for jobs, deals with returned measurements, reports status periodically. start_signal- signal this semaphore when jobs are ready. image_set_start - beginning image set number to process image_set_end - last image set number to process overwrite - whether to recompute imagesets that already have data in initial_measurements. ''' from javabridge import attach, detach posted_analysis_started = False acknowledged_thread_start = False measurements = None workspace = None attach() try: # listen for pipeline events, and pass them upstream self.pipeline.add_listener(lambda pipe, evt: self.post_event(evt)) initial_measurements = None if self.output_path is None: # Caller wants a temporary measurements file. fd, filename = tempfile.mkstemp( ".h5", dir=cpprefs.get_temporary_directory()) try: fd = os.fdopen(fd, "wb") fd.write(self.initial_measurements_buf) fd.close() initial_measurements = cpmeas.Measurements( filename=filename, mode="r") measurements = cpmeas.Measurements( image_set_start=None, copy=initial_measurements, mode="a") finally: if initial_measurements is not None: initial_measurements.close() os.unlink(filename) else: with open(self.output_path, "wb") as fd: fd.write(self.initial_measurements_buf) measurements = cpmeas.Measurements(image_set_start=None, filename=self.output_path, mode="a") # The shared dicts are needed in jobserver() self.shared_dicts = [m.get_dictionary() for m in self.pipeline.modules()] workspace = cpw.Workspace(self.pipeline, None, None, None, measurements, cpimage.ImageSetList()) if image_set_end is None: image_set_end = measurements.get_image_numbers()[-1] image_sets_to_process = filter( lambda x: x >= image_set_start and x <= image_set_end, measurements.get_image_numbers()) self.post_event(AnalysisStarted()) posted_analysis_started = True # reset the status of every image set that needs to be processed has_groups = measurements.has_groups() if self.pipeline.requires_aggregation(): overwrite = True if has_groups and not overwrite: if not measurements.has_feature(cpmeas.IMAGE, self.STATUS): overwrite = True else: group_status = {} for image_number in measurements.get_image_numbers(): group_number = measurements[ cpmeas.IMAGE, cpmeas.GROUP_NUMBER, image_number] status = measurements[cpmeas.IMAGE, self.STATUS, image_number] if status != self.STATUS_DONE: group_status[group_number] = self.STATUS_UNPROCESSED elif group_number not in group_status: group_status[group_number] = self.STATUS_DONE new_image_sets_to_process = [] for image_set_number in image_sets_to_process: needs_reset = False if (overwrite or (not measurements.has_measurements( cpmeas.IMAGE, self.STATUS, image_set_number)) or (measurements[cpmeas.IMAGE, self.STATUS, image_set_number] != self.STATUS_DONE)): needs_reset = True elif has_groups: group_number = measurements[ cpmeas.IMAGE, cpmeas.GROUP_NUMBER, image_set_number] if group_status[group_number] != self.STATUS_DONE: needs_reset = True if needs_reset: measurements[cpmeas.IMAGE, self.STATUS, image_set_number] = \ self.STATUS_UNPROCESSED new_image_sets_to_process.append(image_set_number) image_sets_to_process = new_image_sets_to_process # Find image groups. These are written into measurements prior to # analysis. Groups are processed as a single job. if has_groups or self.pipeline.requires_aggregation(): worker_runs_post_group = True job_groups = {} for image_set_number in image_sets_to_process: group_number = measurements[cpmeas.IMAGE, cpmeas.GROUP_NUMBER, image_set_number] group_index = measurements[cpmeas.IMAGE, cpmeas.GROUP_INDEX, image_set_number] job_groups[group_number] = job_groups.get(group_number, []) + [(group_index, image_set_number)] job_groups = [[isn for _, isn in sorted(job_groups[group_number])] for group_number in sorted(job_groups)] else: worker_runs_post_group = False # prepare_group will be run in worker, but post_group is below. job_groups = [[image_set_number] for image_set_number in image_sets_to_process] # XXX - check that any constructed groups are complete, i.e., # image_set_start and image_set_end shouldn't carve them up. if not worker_runs_post_group: # put the first job in the queue, then wait for the first image to # finish (see the check of self.finish_queue below) to post the rest. # This ensures that any shared data from the first imageset is # available to later imagesets. self.work_queue.put((job_groups[0], worker_runs_post_group, True)) waiting_for_first_imageset = True del job_groups[0] else: waiting_for_first_imageset = False for job in job_groups: self.work_queue.put((job, worker_runs_post_group, False)) job_groups = [] start_signal.release() acknowledged_thread_start = True # We loop until every image is completed, or an outside event breaks the loop. while not self.cancelled: # gather measurements while not self.received_measurements_queue.empty(): image_numbers, buf = self.received_measurements_queue.get() image_numbers = [int(i) for i in image_numbers] recd_measurements = cpmeas.load_measurements_from_buffer(buf) self.copy_recieved_measurements(recd_measurements, measurements, image_numbers) recd_measurements.close() del recd_measurements # check for jobs in progress while not self.in_process_queue.empty(): image_set_numbers = self.in_process_queue.get() for image_set_number in image_set_numbers: measurements[cpmeas.IMAGE, self.STATUS, int(image_set_number)] = self.STATUS_IN_PROCESS # check for finished jobs that haven't returned measurements, yet while not self.finished_queue.empty(): finished_req = self.finished_queue.get() measurements[ cpmeas.IMAGE, self.STATUS, int(finished_req.image_set_number)] = self.STATUS_FINISHED_WAITING if waiting_for_first_imageset: assert isinstance(finished_req, ImageSetSuccessWithDictionary) self.shared_dicts = finished_req.shared_dicts waiting_for_first_imageset = False assert len(self.shared_dicts) == len(self.pipeline.modules()) # if we had jobs waiting for the first image set to finish, # queue them now that the shared state is available. for job in job_groups: self.work_queue.put((job, worker_runs_post_group, False)) finished_req.reply(Ack()) # check progress and report counts = collections.Counter(measurements[cpmeas.IMAGE, self.STATUS, image_set_number] for image_set_number in image_sets_to_process) self.post_event(AnalysisProgress(counts)) # Are we finished? if counts[self.STATUS_DONE] == len(image_sets_to_process): last_image_number = measurements.get_image_numbers()[-1] measurements.image_set_number = last_image_number if not worker_runs_post_group: self.pipeline.post_group(workspace, {}) workspace = cpw.Workspace(self.pipeline, None, None, None, measurements, None, None) workspace.post_run_display_handler = \ self.post_run_display_handler self.pipeline.post_run(workspace) break measurements.flush() # not done, wait for more work with self.interface_work_cv: while (self.paused or ((not self.cancelled) and self.in_process_queue.empty() and self.finished_queue.empty() and self.received_measurements_queue.empty())): self.interface_work_cv.wait() # wait for a change of status or work to arrive finally: detach() # Note - the measurements file is owned by the queue consumer # after this post_event. # if not acknowledged_thread_start: start_signal.release() if posted_analysis_started: was_cancelled = self.cancelled self.post_event(AnalysisFinished(measurements, was_cancelled)) self.stop_workers() self.analysis_id = False # this will cause the jobserver thread to exit
print('big endian:', meta.getPixelsBigEndian(0, 0)) print('dim order:', meta.getPixelsDimensionOrder(0, 0)) print('pixel type:', meta.getPixelsPixelType(0, 0)) print('size x:', meta.getPixelsSizeX(0, 0)) print('size y:', meta.getPixelsSizeY(0, 0)) print('size c:', meta.getPixelsSizeC(0, 0)) print('size z:', meta.getPixelsSizeZ(0, 0)) print('size t:', meta.getPixelsSizeT(0, 0)) print('samples per pixel:', meta.getLogicalChannelSamplesPerPixel(0, 0)) writer.setMetadataRetrieve(meta) writer.setId(out_file) for image in images: if len(image.shape) == 3 and image.shape[2] == 3: save_im = np.array( [image[:, :, 0], image[:, :, 1], image[:, :, 2]]).astype(np.uint8).flatten() else: save_im = image.astype(np.uint8).flatten() writer.saveBytes(env.make_byte_array(save_im), (image is images[-1])) writer.close() print('Done writing image :)') # import PIL.Image as Image # im = Image.open(out_file, 'r') # im.show() jutil.detach() app.MainLoop()
def __init__(self, path, **kwargs): # noqa """ Initialize the tile class. See the base class for other available parameters. :param path: the associated file path. :param maxSize: either a number or an object with {'width': (width), 'height': height} in pixels. If None, the default max size is used. """ super().__init__(path, **kwargs) self._logger = config.getConfig('logger') largeImagePath = self._getLargeImagePath() ext = os.path.splitext(largeImagePath)[1] if not ext: raise TileSourceException( 'File cannot be opened via bioformats because it has no ' 'extension to specify the file type (%s).' % largeImagePath) if ext.lower() in ('.jpg', '.jpeg', '.jpe', '.png'): raise TileSourceException('File will not be opened by bioformats reader') if not _startJavabridge(self._logger): raise TileSourceException( 'File cannot be opened by bioformats reader because javabridge failed to start') self._tileLock = threading.RLock() try: javabridge.attach() try: self._bioimage = bioformats.ImageReader(largeImagePath) except AttributeError as exc: self._logger.debug('File cannot be opened via Bioformats. (%r)' % exc) raise TileSourceException('File cannot be opened via Bioformats. (%r)' % exc) _openImages.append(self) rdr = self._bioimage.rdr # Bind additional functions not done by bioformats module. # Functions are listed at https://downloads.openmicroscopy.org # //bio-formats/5.1.5/api/loci/formats/IFormatReader.html for (name, params, desc) in [ ('getBitsPerPixel', '()I', 'Get the number of bits per pixel'), ('getEffectiveSizeC', '()I', 'effectiveC * Z * T = imageCount'), ('isNormalized', '()Z', 'Is float data normalized'), ('isMetadataComplete', '()Z', 'True if metadata is completely parsed'), ('getDomains', '()[Ljava/lang/String;', 'Get a list of domains'), ('getZCTCoords', '(I)[I', 'Gets the Z, C and T coordinates ' '(real sizes) corresponding to the given rasterized index value.'), ('getOptimalTileWidth', '()I', 'the optimal sub-image width ' 'for use with openBytes'), ('getOptimalTileHeight', '()I', 'the optimal sub-image height ' 'for use with openBytes'), ('getResolutionCount', '()I', 'The number of resolutions for ' 'the current series'), ('setResolution', '(I)V', 'Set the resolution level'), ('getResolution', '()I', 'The current resolution level'), ('hasFlattenedResolutions', '()Z', 'True if resolutions have been flattened'), ('setFlattenedResolutions', '(Z)V', 'Set if resolution should be flattened'), ]: setattr(rdr, name, types.MethodType( javabridge.jutil.make_method(name, params, desc), rdr)) # rdr.setFlattenedResolutions(False) self._metadata = { 'dimensionOrder': rdr.getDimensionOrder(), 'metadata': javabridge.jdictionary_to_string_dictionary( rdr.getMetadata()), 'seriesMetadata': javabridge.jdictionary_to_string_dictionary( rdr.getSeriesMetadata()), 'seriesCount': rdr.getSeriesCount(), 'imageCount': rdr.getImageCount(), 'rgbChannelCount': rdr.getRGBChannelCount(), 'sizeColorPlanes': rdr.getSizeC(), 'sizeT': rdr.getSizeT(), 'sizeZ': rdr.getSizeZ(), 'sizeX': rdr.getSizeX(), 'sizeY': rdr.getSizeY(), 'pixelType': rdr.getPixelType(), 'isLittleEndian': rdr.isLittleEndian(), 'isRGB': rdr.isRGB(), 'isInterleaved': rdr.isInterleaved(), 'isIndexed': rdr.isIndexed(), 'bitsPerPixel': rdr.getBitsPerPixel(), 'sizeC': rdr.getEffectiveSizeC(), 'normalized': rdr.isNormalized(), 'metadataComplete': rdr.isMetadataComplete(), # 'domains': rdr.getDomains(), 'optimalTileWidth': rdr.getOptimalTileWidth(), 'optimalTileHeight': rdr.getOptimalTileHeight(), 'resolutionCount': rdr.getResolutionCount(), 'flattenedResolutions': rdr.hasFlattenedResolutions(), } self._checkSeries(rdr) bmd = bioformats.metadatatools.MetadataRetrieve(self._bioimage.metadata) try: self._metadata['channelNames'] = [ bmd.getChannelName(0, c) or bmd.getChannelID(0, c) for c in range(self._metadata['sizeColorPlanes'])] except Exception: self._metadata['channelNames'] = [] for key in ['sizeXY', 'sizeC', 'sizeZ', 'sizeT']: if not isinstance(self._metadata[key], int) or self._metadata[key] < 1: self._metadata[key] = 1 self.sizeX = self._metadata['sizeX'] self.sizeY = self._metadata['sizeY'] self._computeTiles() self._computeLevels() self._computeMagnification() except javabridge.JavaException as exc: es = javabridge.to_string(exc.throwable) self._logger.debug('File cannot be opened via Bioformats. (%s)' % es) raise TileSourceException('File cannot be opened via Bioformats. (%s)' % es) except AttributeError: self._logger.exception('The bioformats reader threw an unhandled exception.') raise TileSourceException('The bioformats reader threw an unhandled exception.') finally: if javabridge.get_env(): javabridge.detach() if self.levels < 1: raise TileSourceException( 'OpenSlide image must have at least one level.') if self.sizeX <= 0 or self.sizeY <= 0: raise TileSourceException('Bioformats tile size is invalid.')
def __init__(self, df, penaltydiscount = 4, depth = 3, faithfulness = True, verbose = False, java_max_heap_size = None): tetrad_libdir = os.path.join(os.path.dirname(__file__), 'lib') # print 'tetrad_libdir: %s' % tetrad_libdir for l in glob.glob(tetrad_libdir + os.sep + "*.jar"): javabridge.JARS.append(str(l)) javabridge.start_vm(run_headless=True, max_heap_size = java_max_heap_size) javabridge.attach() tetradData = None if(len(df.index)*df.columns.size <= 1500): node_list = javabridge.JWrapper(javabridge.make_instance("java/util/ArrayList", "()V")) for col in df.columns: nodname = javabridge.make_instance("java/lang/String", "(Ljava/lang/String;)V",col) nodi = javabridge.make_instance("edu/cmu/tetrad/graph/GraphNode", "(Ljava/lang/String;)V",nodname) node_list.add(nodi) tetradMatrix = javabridge.JWrapper(javabridge.make_instance("edu/cmu/tetrad/util/TetradMatrix","(II)V", len(df.index),df.columns.size)) for row in df.index: for col in range(0,df.columns.size): tetradMatrix.set(row,col,df.ix[row][col]) tetradData = javabridge.static_call("edu/cmu/tetrad/data/ColtDataSet","makeContinuousData", "(Ljava/util/List;Ledu/cmu/tetrad/util/TetradMatrix;)Ledu/cmu/tetrad/data/DataSet;", node_list,tetradMatrix) else: #Generate random name temp_data_file = ''.join(random.choice(string.lowercase) for i in range(10)) + '.csv' temp_data_path = os.path.join(tempfile.gettempdir(), temp_data_file) df.to_csv(temp_data_path, sep = "\t") f = javabridge.make_instance("java/io/File", "(Ljava/lang/String;)V",temp_data_path) excludeVar = javabridge.JWrapper(javabridge.make_instance("java/util/HashSet","()V")) excludeVar.add("MULT") tetradData = javabridge.static_call("edu/cmu/tetrad/data/BigDataSetUtility","readInContinuousData","(Ljava/io/File;CLjava/util/Set;)Ledu/cmu/tetrad/data/DataSet;",f,"\t",excludeVar) os.remove(temp_data_path) fgs = javabridge.make_instance("edu/cmu/tetrad/search/Fgs","(Ledu/cmu/tetrad/data/DataSet;)V",tetradData) fgs = javabridge.JWrapper(fgs) fgs.setPenaltyDiscount(penaltydiscount)# set to 2 if variable# <= 50 otherwise set it to 4 fgs.setDepth(depth)#-1 fgs.setNumPatternsToStore(0) fgs.setFaithfulnessAssumed(faithfulness) fgs.setVerbose(verbose) tetradGraph = fgs.search() graph = pydot.Dot(graph_type='digraph') n = tetradGraph.getNodeNames().toString() n = n[1:len(n)-1] n = n.split(",") nodes = [] for i in range(0,len(n)): node = n[i] n[i] = node.strip() nodes.append(pydot.Node(n[i])) graph.add_node(nodes[i]) self.nodes = n e = tetradGraph.getEdges().toString() e = e[1:len(e)-1] e = e.split(",") for i in range(0,len(e)): e[i] = e[i].strip() token = e[i].split(" ") if(len(token) == 3): src = token[0] arc = token[1] dst = token[2] if(pycausal.isNodeExisting(n,src) and pycausal.isNodeExisting(n,dst)): edge = pydot.Edge(nodes[n.index(src)],nodes[n.index(dst)]) if(arc == "---"): edge.set_arrowhead("none") graph.add_edge(edge) self.edges = e javabridge.detach() javabridge.kill_vm() self.graph = graph
def kill_vm(): javabridge.detach() javabridge.kill_vm()
def __init__(self, df, penaltydiscount=4, depth=3, faithfulness=True, verbose=False, java_max_heap_size=None): tetrad_libdir = os.path.join(os.path.dirname(__file__), 'lib') # print 'tetrad_libdir: %s' % tetrad_libdir for l in glob.glob(tetrad_libdir + os.sep + "*.jar"): javabridge.JARS.append(str(l)) javabridge.start_vm(run_headless=True, max_heap_size=java_max_heap_size) javabridge.attach() tetradData = None if (len(df.index) * df.columns.size <= 1500): node_list = javabridge.JWrapper( javabridge.make_instance("java/util/ArrayList", "()V")) for col in df.columns: nodname = javabridge.make_instance("java/lang/String", "(Ljava/lang/String;)V", col) nodi = javabridge.make_instance( "edu/cmu/tetrad/graph/GraphNode", "(Ljava/lang/String;)V", nodname) node_list.add(nodi) tetradMatrix = javabridge.JWrapper( javabridge.make_instance("edu/cmu/tetrad/util/TetradMatrix", "(II)V", len(df.index), df.columns.size)) for row in df.index: for col in range(0, df.columns.size): tetradMatrix.set(row, col, df.ix[row][col]) tetradData = javabridge.static_call( "edu/cmu/tetrad/data/ColtDataSet", "makeContinuousData", "(Ljava/util/List;Ledu/cmu/tetrad/util/TetradMatrix;)Ledu/cmu/tetrad/data/DataSet;", node_list, tetradMatrix) else: #Generate random name temp_data_file = ''.join( random.choice(string.lowercase) for i in range(10)) + '.csv' temp_data_path = os.path.join(tempfile.gettempdir(), temp_data_file) df.to_csv(temp_data_path, sep="\t") f = javabridge.make_instance("java/io/File", "(Ljava/lang/String;)V", temp_data_path) excludeVar = javabridge.JWrapper( javabridge.make_instance("java/util/HashSet", "()V")) excludeVar.add("MULT") tetradData = javabridge.static_call( "edu/cmu/tetrad/data/BigDataSetUtility", "readInContinuousData", "(Ljava/io/File;CLjava/util/Set;)Ledu/cmu/tetrad/data/DataSet;", f, "\t", excludeVar) os.remove(temp_data_path) fgs = javabridge.make_instance("edu/cmu/tetrad/search/Fgs", "(Ledu/cmu/tetrad/data/DataSet;)V", tetradData) fgs = javabridge.JWrapper(fgs) fgs.setPenaltyDiscount( penaltydiscount ) # set to 2 if variable# <= 50 otherwise set it to 4 fgs.setDepth(depth) #-1 fgs.setNumPatternsToStore(0) fgs.setFaithfulnessAssumed(faithfulness) fgs.setVerbose(verbose) tetradGraph = fgs.search() graph = pydot.Dot(graph_type='digraph') n = tetradGraph.getNodeNames().toString() n = n[1:len(n) - 1] n = n.split(",") nodes = [] for i in range(0, len(n)): node = n[i] n[i] = node.strip() nodes.append(pydot.Node(n[i])) graph.add_node(nodes[i]) self.nodes = n e = tetradGraph.getEdges().toString() e = e[1:len(e) - 1] e = e.split(",") for i in range(0, len(e)): e[i] = e[i].strip() token = e[i].split(" ") if (len(token) == 3): src = token[0] arc = token[1] dst = token[2] if (pycausal.isNodeExisting(n, src) and pycausal.isNodeExisting(n, dst)): edge = pydot.Edge(nodes[n.index(src)], nodes[n.index(dst)]) if (arc == "---"): edge.set_arrowhead("none") graph.add_edge(edge) self.edges = e javabridge.detach() javabridge.kill_vm() self.graph = graph