def _getAssociatedImage(self, imageKey):
"""
Get an associated image in PIL format.
:param imageKey: the key of the associated image.
:return: the image in PIL format or None.
"""
info = self._metadata['seriesAssociatedImages'].get(imageKey)
if info is None:
return
series = info['seriesNum']
with self._tileLock:
try:
javabridge.attach()
image = self._bioimage.read(
series=series,
rescale=False, # return internal data types
XYWH=(0, 0, info['sizeX'], info['sizeY']))
except javabridge.JavaException as exc:
es = javabridge.to_string(exc.throwable)
raise TileSourceException('Failed to get Bioformat series (%s, %r).' % (es, (
series, info['sizeX'], info['sizeY'])))
finally:
if javabridge.get_env():
javabridge.detach()
return large_image.tilesource.base._imageToPIL(image)
def setUpClass(self):
from cellprofiler.modules.tests import \
example_images_directory, maybe_download_example_image
self.root_dir = example_images_directory()
self.file_name = maybe_download_example_image(["ExampleSBSImages"],
"Channel1-01-A-01.tif")
J.attach()
def setUpClass(cls):
from cellprofiler.modules.tests \
import example_images_directory, maybe_download_sbs
maybe_download_sbs()
cls.root_dir = example_images_directory()
J.attach()
def enter_thread(self):
J.attach()
if not cpprefs.get_awt_headless():
J.activate_awt()
self.notify_socket = the_zmq_context.socket(zmq.SUB)
self.notify_socket.setsockopt(zmq.SUBSCRIBE, "")
self.notify_socket.connect(NOTIFY_ADDR)
def write_tile(br, bw, X, Y, Z, C):
"""Read and write one tile of a multichannel image
This function is intended to be run inside of a thread. Since data is sent
to the JVM and the GIL is released, reading images can be performed in
parallel. Writing still requires a lock.
Args:
br ([bfio.BioReader]): BioReader to be read from
bw ([bfio.BioWriter]): BioWriter to be written to
X ([list]): Range of x-values to read/write
Y ([list]): Range of y-values to grab/write
C ([list]): Channel to write
"""
# Connect to the JVM
jutil.attach()
# Read the image
image = br.read_image(X=X, Y=Y, Z=Z)
# Lock the thread and write the image
with lock:
bw.write_image(image, X=[X[0]], Y=[Y[0]], Z=[Z[0]], C=C)
# Detach from the JVM
jutil.detach()
def validation_queue_handler():
from javabridge import attach, detach
attach()
try:
while mv_constants.validation_queue_keep_running:
request = mv_constants.validation_queue.get()
if (
not isinstance(request, ValidationRequestController)
or request.cancelled
):
continue
start = time.perf_counter()
try:
validate_module(
request.pipeline, request.module_num, request.callback,
)
except:
pass
# Make sure this thread utilizes less than 1/2 of GIL clock
wait_for = max(0.25, time.perf_counter() - start)
time.sleep(wait_for)
finally:
detach()
logging.info("Exiting the pipeline validation thread")
def __init__(self, imgfile, output_path):
self.imgfile = imgfile
try:
self.image = I.ImarisImage(self.imgfile)
if not os.path.isdir(output_path):
raise IOError('Invalid output directory: ', output_path)
else:
# create unique output directory from image filename
filename = basename(imgfile)
new_folder = splitext(filename)[0]
image_folder = join(output_path, new_folder)
if not os.path.isdir(image_folder):
os.makedirs(image_folder)
self.output_folder = image_folder
try:
javabridge.start_vm(run_headless=True, class_path=bioformats.JARS)
javabridge.attach()
except Exception as e:
raise (e)
javabridge.attach()
self.basicmeta = javabridge.jutil.to_string(
bioformats.formatreader.make_iformat_reader_class().getMetadata(bioformats.ImageReader(imgfile).rdr)).split(',')
self.basicmeta = sorted(self.basicmeta)
self.omemeta = bioformats.get_omexml_metadata(imgfile).split("></")
self.omemeta1 = self.omemeta[0].split(".xsd")
self.omemeta2 = self.omemeta[1:]
#javabridge.detach()
except Exception as e:
raise (e)
def setUpClass(cls):
from cellprofiler.modules.tests \
import example_images_directory, maybe_download_sbs
maybe_download_sbs()
cls.root_dir = example_images_directory()
J.attach()
def start_vm(args=None,
class_path=None,
max_heap_size=None,
run_headless=False):
"""
Start the JVM via Javabridge.
"""
if class_path is None:
class_path = get_java_class_path()
# Start the JVM.
javabridge.start_vm(args=args,
class_path=class_path,
max_heap_size=max_heap_size,
run_headless=run_headless)
# Suppress Java logging to stdout.
java_stack = javabridge.make_instance('java/io/ByteArrayOutputStream',
"()V")
java_stack_ps = javabridge.make_instance('java/io/PrintStream',
"(Ljava/io/OutputStream;)V",
java_stack)
javabridge.static_call('Ljava/lang/System;', "setErr",
'(Ljava/io/PrintStream;)V', java_stack_ps)
java_out = javabridge.make_instance('java/io/ByteArrayOutputStream', "()V")
java_out_ps = javabridge.make_instance('java/io/PrintStream',
"(Ljava/io/OutputStream;)V",
java_out)
javabridge.static_call('Ljava/lang/System;', "setOut",
'(Ljava/io/PrintStream;)V', java_out_ps)
javabridge.attach()
def setUpClass(self):
from cellprofiler.modules.tests import \
example_images_directory, maybe_download_example_image
self.root_dir = example_images_directory()
self.file_name = maybe_download_example_image(
["ExampleSBSImages"], "Channel1-01-A-01.tif")
J.attach()
def enter_thread(self):
J.attach()
if not cpprefs.get_awt_headless():
J.activate_awt()
self.notify_socket = the_zmq_context.socket(zmq.SUB)
self.notify_socket.setsockopt(zmq.SUBSCRIBE, "")
self.notify_socket.connect(NOTIFY_ADDR)
def OnInit(self):
"""Initialize CPA
"""
"""List of tables created by the user during this session"""
self.user_tables = []
# splashscreen
splashimage = cpa.icons.cpa_splash.ConvertToBitmap()
# If the splash image has alpha, it shows up transparently on
# windows, so we blend it into a white background.
splashbitmap = wx.EmptyBitmapRGBA(splashimage.GetWidth(), splashimage.GetHeight(), 255, 255, 255, 255)
dc = wx.MemoryDC()
dc.SelectObject(splashbitmap)
dc.DrawBitmap(splashimage, 0, 0)
dc.Destroy() # necessary to avoid a crash in splashscreen
splash = wx.SplashScreen(splashbitmap, wx.SPLASH_CENTRE_ON_SCREEN | wx.SPLASH_TIMEOUT, 2000, None, -1)
self.splash = splash
p = Properties.getInstance()
if not p.is_initialized():
from cpa.guiutils import show_load_dialog
splash.Destroy()
if not show_load_dialog():
logging.error("CellProfiler Analyst requires a properties file. Exiting.")
return False
self.frame = MainGUI(p, None, size=(860, -1))
self.frame.Show(True)
db = cpa.dbconnect.DBConnect.getInstance()
# Black magic: Bus errors occur on Mac OS X if we wait until
# the JVM or the wx event look has started to connect. But it
# has to be done after we have read the properties file. So we
# do it here.
db.connect()
db.register_gui_parent(self.frame)
# The JVM has to be started after db.connect(), otherwise bus errors
# occur on Mac OS X.
javabridge.start_vm(class_path=bioformats.JARS, run_headless=False)
javabridge.attach()
javabridge.activate_awt()
try:
if __version__ != -1:
import cellprofiler.utilities.check_for_updates as cfu
cfu.check_for_updates(
"http://cellprofiler.org/CPAupdate.html",
max(__version__, cpaprefs.get_skip_version()),
new_version_cb,
user_agent="CPAnalyst/2.0.%d" % (__version__),
)
except ImportError:
logging.warn(
"CPA was unable to check for updates. Could not import cellprofiler.utilities.check_for_updates."
)
return True
def start_vm(self, java_max_heap_size = None):
tetrad_libdir = os.path.join(os.path.dirname(__file__), 'lib')
for l in glob.glob(tetrad_libdir + os.sep + "*.jar"):
javabridge.JARS.append(str(l))
javabridge.start_vm(run_headless=True, max_heap_size = java_max_heap_size)
javabridge.attach()
def start_vm(self, java_max_heap_size = None):
tetrad_libdir = os.path.join(os.path.dirname(__file__), 'lib')
for l in glob.glob(tetrad_libdir + os.sep + "*.jar"):
javabridge.JARS.append(str(l))
javabridge.start_vm(run_headless=True, max_heap_size = java_max_heap_size)
javabridge.attach()
def __call__(self, id, filename, uri, **kwargs):
"""
Extract metadata from a Datafile using the get_meta function and save
the outputs as DatafileParameters. This function differs from
process_meta in that it generates an output directory in the metadata
store and passes it to the metadata processing func so that outputs
e.g., preview images or metadata files) can be saved.
param id: Datafile ID
type id: integer
param filename: Absolute path to a file for processing
type filename: string
param uri: Dataset URI
type uri: string
param kwargs: Extra arguments
type kwargs: object
return Extracted metadata
rtype dict
"""
_, extension = os.path.splitext(filename)
if extension[1:] not in bioformats.READABLE_FORMATS:
return None
logger.info("Applying Bioformats filter to {}...".format(filename))
# Need to start a JVM in each thread
check_and_start_jvm()
try:
javabridge.attach()
shush_logger()
thumb_rel_path, thumb_abs_path = get_thumbnail_paths(
id, filename, uri)
if not os.path.exists(os.path.dirname(thumb_abs_path)):
os.makedirs(os.path.dirname(thumb_abs_path))
rsp = get_meta(filename, os.path.dirname(thumb_abs_path), **kwargs)
if rsp is not None:
metadata = []
for i in rsp:
metadata.append(self.filter_metadata(i))
return metadata[0]
except Exception as e:
logger.error(str(e))
logger.debug(traceback.format_exc())
finally:
javabridge.detach()
javabridge.kill_vm()
return None
def __del__(self):
if getattr(self, '_bioimage', None) is not None:
try:
javabridge.attach()
self._bioimage.close()
_openImages.remove(self)
finally:
if javabridge.get_env():
javabridge.detach()
def run(self):
javabridge.attach()
try:
self.socket = self.context.socket(zmq.REP)
self.socket.bind(self.address)
logger.info("Binding Knime bridge server to %s" % self.address)
poller = zmq.Poller()
poller.register(self.socket, flags=zmq.POLLIN)
if self.notify_addr is not None:
self.notify_socket = self.context.socket(zmq.SUB)
self.notify_socket.setsockopt(zmq.SUBSCRIBE, "")
self.notify_socket.connect(self.notify_addr)
poller.register(self.notify_socket, flags=zmq.POLLIN)
else:
self.notify_socket = None
start_socket = self.context.socket(zmq.PAIR)
start_socket.connect(self.start_addr)
start_socket.send("OK")
start_socket.close()
try:
while True:
for socket, event in poller.poll(1):
if socket == self.notify_socket:
msg = self.notify_socket.recv_string()
if msg == self.stop_msg:
break
elif socket == self.socket:
msg = self.socket.recv_multipart(copy=False)
session_id_frame = msg.pop(0)
session_id = str(session_id_frame.bytes)
# should be None
wrapper = msg.pop(0)
message_type = msg.pop(0).bytes
if message_type not in self.dispatch:
self.raise_cellprofiler_exception(
session_id,
"Unhandled message type: %s" %
message_type,
)
else:
try:
self.dispatch[message_type](session_id,
message_type,
msg)
except Exception as e:
logger.warn(e.message, exc_info=1)
self.raise_cellprofiler_exception(
session_id, e.message)
else:
continue
break
finally:
if self.notify_socket:
self.notify_socket.close()
self.socket.close()
finally:
javabridge.detach()
def get(self):
total = dm.get_total_object_count()
objKeys = dm.GetRandomObjects(total)
# Start the virtual machine
javabridge.start_vm(class_path=bioformats.JARS, run_headless=True)
javabridge.attach()
javabridge.activate_awt()
return calculateAllJSON(objKeys, base64=True)
def start(class_path=None,
bundled=True,
packages=False,
system_cp=False,
max_heap_size=None):
"""
Initializes the javabridge connection (starts up the JVM).
:param class_path: the additional classpath elements to add
:type class_path: list
:param bundled: whether to add jars from the "lib" directory
:type bundled: bool
:param packages: whether to add jars from Weka packages as well (bool) or an alternative Weka home directory (str)
:type packages: bool or str
:param system_cp: whether to add the system classpath as well
:type system_cp: bool
:param max_heap_size: the maximum heap size (-Xmx parameter, eg 512m or 4g)
:type max_heap_size: str
"""
global started
if started is not None:
logger.info("JVM already running, call jvm.stop() first")
return
# add user-defined jars first
if class_path is not None:
for cp in class_path:
logger.debug("Adding user-supplied classpath=" + cp)
javabridge.JARS.append(cp)
if bundled:
logger.debug("Adding bundled jars")
add_bundled_jars()
if packages is not None:
if isinstance(packages, bool):
if packages:
logger.debug("Adding Weka packages")
add_weka_packages()
else:
logger.debug("Adding Weka packages, using: " + packages)
add_weka_packages(packages)
if system_cp:
logger.debug("Adding system classpath")
add_system_classpath()
logger.debug("Classpath=" + str(javabridge.JARS))
logger.debug("MaxHeapSize=" +
("default" if (max_heap_size is None) else max_heap_size))
javabridge.start_vm(run_headless=True, max_heap_size=max_heap_size)
javabridge.attach()
started = True
def run(self):
javabridge.attach()
try:
self.socket = self.context.socket(zmq.REP)
self.socket.bind(self.address)
logger.info("Binding Knime bridge server to %s" % self.address)
poller = zmq.Poller()
poller.register(self.socket, flags=zmq.POLLIN)
if self.notify_addr is not None:
self.notify_socket = self.context.socket(zmq.SUB)
self.notify_socket.setsockopt(zmq.SUBSCRIBE, "")
self.notify_socket.connect(self.notify_addr)
poller.register(self.notify_socket, flags=zmq.POLLIN)
else:
self.notify_socket = None
start_socket = self.context.socket(zmq.PAIR)
start_socket.connect(self.start_addr)
start_socket.send("OK")
start_socket.close()
try:
while True:
for socket, event in poller.poll(1):
if socket == self.notify_socket:
msg = self.notify_socket.recv_string()
if msg == self.stop_msg:
break
elif socket == self.socket:
msg = self.socket.recv_multipart(copy=False)
session_id_frame = msg.pop(0)
session_id = str(session_id_frame.bytes)
# should be None
wrapper = msg.pop(0)
message_type = msg.pop(0).bytes
if message_type not in self.dispatch:
self.raise_cellprofiler_exception(
session_id,
"Unhandled message type: %s" % message_type)
else:
try:
self.dispatch[message_type](
session_id, message_type, msg)
except Exception, e:
logger.warn(e.message, exc_info=1)
self.raise_cellprofiler_exception(
session_id, e.message)
else:
continue
break
finally:
if self.notify_socket:
self.notify_socket.close()
self.socket.close()
finally:
javabridge.detach()
def get(self):
if cached_json["base64"] == False:
# Start the virtual machine
javabridge.start_vm(class_path=bioformats.JARS, run_headless=True)
javabridge.attach()
javabridge.activate_awt()
# Calculate the Training DataSet and store it
cached_json["json"] = calculateTrainingSetJSON(base64=True)
cached_json["base64"] = True
return cached_json["json"]
def run(self):
if p.force_bioformats:
logging.debug("Starting javabridge")
import bioformats
javabridge.start_vm(class_path=bioformats.JARS, run_headless=True)
javabridge.attach()
try:
while 1:
self.tile_collection.cv.acquire()
# If there are no objects in the queue then wait
while not self.tile_collection.loadq:
self.tile_collection.cv.wait()
if self._want_abort:
self.tile_collection.cv.release()
db = DBConnect()
db.CloseConnection()
logging.info('%s aborted' % self.getName())
return
data = heappop(self.tile_collection.loadq)
obKey = data[1]
display_whole_image = data[
2] #display whole image instead of object image
self.tile_collection.cv.release()
# wait until loading has completed before continuing
with self.tile_collection.load_lock:
# Make sure tile hasn't been deleted outside this thread
if not self.tile_collection.tileData.get(obKey, None):
continue
# Get the tile
new_data = imagetools.FetchTile(
obKey, display_whole_image=display_whole_image)
if new_data is None:
#if fetching fails, leave the tile blank
continue
tile_data = self.tile_collection.tileData.get(obKey, None)
# Make sure tile hasn't been deleted outside this thread
if tile_data is not None:
# copy each channel
for i in range(len(tile_data)):
tile_data[i] = new_data[i]
for window in self.notify_window:
wx.PostEvent(window, TileUpdatedEvent(obKey))
finally:
if javabridge.get_env() is not None:
javabridge.detach()
def start(class_path=None, bundled=True, packages=False, system_cp=False, max_heap_size=None):
"""
Initializes the javabridge connection (starts up the JVM).
:param class_path: the additional classpath elements to add
:type class_path: list
:param bundled: whether to add jars from the "lib" directory
:type bundled: bool
:param packages: whether to add jars from Weka packages as well (bool) or an alternative Weka home directory (str)
:type packages: bool or str
:param system_cp: whether to add the system classpath as well
:type system_cp: bool
:param max_heap_size: the maximum heap size (-Xmx parameter, eg 512m or 4g)
:type max_heap_size: str
"""
global started
if started is not None:
logger.info("JVM already running, call jvm.stop() first")
return
# add user-defined jars first
if class_path is not None:
for cp in class_path:
logger.debug("Adding user-supplied classpath=" + cp)
javabridge.JARS.append(cp)
if bundled:
logger.debug("Adding bundled jars")
add_bundled_jars()
if packages is not None:
if isinstance(packages, bool):
if packages:
logger.debug("Adding Weka packages")
add_weka_packages()
else:
logger.debug("Adding Weka packages, using: " + packages)
add_weka_packages(packages)
if system_cp:
logger.debug("Adding system classpath")
add_system_classpath()
logger.debug("Classpath=" + str(javabridge.JARS))
logger.debug("MaxHeapSize=" + ("default" if (max_heap_size is None) else max_heap_size))
javabridge.start_vm(run_headless=True, max_heap_size=max_heap_size)
javabridge.attach()
started = True
def setUpModule():
try:
os.environ['PYJNIUS_ACTIVATE']
except KeyError:
import javabridge
try:
# include user added classpath
classpath = os.environ['CLASSPATH']
classpath = classpath.split(os.pathsep)
javabridge.JARS.extend(classpath)
except KeyError:
None
javabridge.start_vm()
javabridge.attach()
def engine(q_request, q_response):
logger.info("Starting script engine thread")
J.attach()
while True:
try:
msg = J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
q_request.take();""", dict(q_request=q_request))
if logger.level <= logging.INFO:
logger.info("Received engine request: %s", J.to_string(msg))
payload = J.get_collection_wrapper(
J.run_script("msg.payload", dict(msg=msg)))
if J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
msg.command==CPythonScriptEngine.EngineCommands.EXECUTE;
""", dict(msg=msg)):
response = do_execute(payload)
elif J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
msg.command==CPythonScriptEngine.EngineCommands.EVALUATE;
""", dict(msg=msg)):
response = do_evaluate(payload)
elif J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
msg.command==CPythonScriptEngine.EngineCommands.CLOSE_ENGINE;
""", dict(msg=msg)):
logger.info("Exiting script engine thread after close request")
break
else:
logger.warn(
"Received unknown command: %s" %
J.run_script("msg.command.toString()", dict(msg=msg)))
response = J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
var exception = new java.lang.RuntimeException(
java.lang.String.format('Unknown command: %s', msg.command.toString()));
var payload = new java.util.ArrayList();
payload.add(exception);
new CPythonScriptEngine.Message(
CPythonScriptEngine.EngineCommands.EXCEPTION, payload);
""", dict(msg=msg))
J.run_script("q_response.put(response);",
dict(q_response=q_response, response=response))
except:
# To do: how to handle failure, probably from .take()
# Guessing that someone has managed to interrupt our thread
logger.warn("Exiting script engine thread", exc_info=True)
J.detach()
def setUpModule():
try:
os.environ["PYJNIUS_ACTIVATE"]
except KeyError:
import javabridge
try:
# include user added classpath
classpath = os.environ["CLASSPATH"]
classpath = classpath.split(os.pathsep)
javabridge.JARS.extend(classpath)
except KeyError:
None
javabridge.start_vm()
javabridge.attach()
def _performInit(self):
# Set the classpath
javaclasspath = [
os.path.realpath(
os.path.join(os.path.dirname(__file__), '..', '..', 'lib',
'saiku-report-viewer-server.jar'))
]
# Start JVM
javabridge.start_vm(class_path=[os.pathsep.join(javaclasspath)],
run_headless=True)
javabridge.attach()
# Init the Saiku Report Server Implementation
self.reportServerImpl = javabridge.JClassWrapper(
'org.saiku.reportviewer.server.api.ReportServerImpl')()
self.reportServerImpl.init()
def start_vm( args=None, class_path=None, max_heap_size=None, run_headless=False ):
"""
Start the JVM via Javabridge.
"""
if class_path is None:
class_path = get_java_class_path()
# Start the JVM.
javabridge.start_vm( args=args, class_path=class_path, max_heap_size=max_heap_size, run_headless=run_headless )
# Suppress Java logging to stdout.
java_stack = javabridge.make_instance( 'java/io/ByteArrayOutputStream', "()V" )
java_stack_ps = javabridge.make_instance( 'java/io/PrintStream', "(Ljava/io/OutputStream;)V", java_stack )
javabridge.static_call( 'Ljava/lang/System;', "setErr", '(Ljava/io/PrintStream;)V', java_stack_ps )
java_out = javabridge.make_instance( 'java/io/ByteArrayOutputStream', "()V" )
java_out_ps = javabridge.make_instance( 'java/io/PrintStream', "(Ljava/io/OutputStream;)V", java_out )
javabridge.static_call( 'Ljava/lang/System;', "setOut", '(Ljava/io/PrintStream;)V', java_out_ps )
javabridge.attach()
def _read_image(self, filename_or_url, log_io=True):
# The opener's destructor deletes the temprary files, so the
# opener must not be GC'ed until the image has been loaded.
opener = ThrowingURLopener()
if p.image_url_prepend:
parsed = urllib.parse.urlparse(p.image_url_prepend +
filename_or_url)
if parsed.scheme:
try:
filename_or_url, ignored_headers = opener.retrieve(
parsed.geturl())
except IOError as e:
if e.args[0] == 'http error':
status_code, message = e.args[1:3]
raise ClearException(
'Failed to load image from %s' % parsed.geturl(),
'%d %s' % (status_code, message))
else:
raise
if not p.force_bioformats and os.path.splitext(
filename_or_url)[-1].lower() in IMAGEIO_FORMATS:
if log_io:
logging.info('ImageIO: Loading image from "%s"' %
filename_or_url)
try:
return imageio.imread(filename_or_url)
except FileNotFoundError:
logging.error(f"File not found: {filename_or_url}")
return
except:
logging.info(
'Loading with ImageIO failed, falling back to BioFormats')
if javabridge.get_env() is None:
logging.debug("Starting javabridge")
javabridge.start_vm(class_path=bioformats.JARS, run_headless=True)
javabridge.attach()
if log_io:
logging.info('BioFormats: Loading image from "%s"' %
filename_or_url)
try:
return bioformats.load_image(filename_or_url, rescale=False)
except FileNotFoundError:
logging.error(f"File not found: {filename_or_url}")
except:
logging.error(
'Loading with BioFormats failed, file may be corrupted or unavailable'
)
def _monitor_thread():
main_thread = threading.main_thread()
main_thread.join()
if len(_openImages):
try:
javabridge.attach()
while len(_openImages):
source = _openImages.pop()
try:
source._bioimage.close()
except Exception:
pass
source._bioimage = None
finally:
if javabridge.get_env():
javabridge.detach()
_stopJavabridge()
def __init__(self, path, heap_size=4):
self.path = path
self.filename = os.path.basename(self.path)
self.heap_size = heap_size # GB
javabridge.start_vm(class_path=javabridge.JARS + JARS,run_headless=True,max_heap_size=f'{self.heap_size}G')
javabridge.attach()
DebugTools = javabridge.JClassWrapper("loci.common.DebugTools")
DebugTools.setRootLevel("OFF")
self.rdr = _formatreader.ImageReader(self.path, perform_init=True)
self.reader = self.rdr.rdr
self.J = javabridge.JWrapper(self.reader.getMetadataStore())
self.ids = []
self.datas = []
def run(self):
javabridge.attach()
try:
while 1:
self.tile_collection.cv.acquire()
# If there are no objects in the queue then wait
while not self.tile_collection.loadq:
self.tile_collection.cv.wait()
if self._want_abort:
self.tile_collection.cv.release()
logging.info('%s aborted' % self.getName())
return
data = heappop(self.tile_collection.loadq)
obKey = data[1]
display_whole_image = data[
2] #display whole image instead of object image
self.tile_collection.cv.release()
# wait until loading has completed before continuing
with self.tile_collection.load_lock:
# Make sure tile hasn't been deleted outside this thread
if not self.tile_collection.tileData.get(obKey, None):
continue
# Get the tile
new_data = imagetools.FetchTile(
obKey, display_whole_image=display_whole_image)
if new_data is None:
#if fetching fails, leave the tile blank
continue
tile_data = self.tile_collection.tileData.get(obKey, None)
# Make sure tile hasn't been deleted outside this thread
if tile_data is not None:
# copy each channel
for i in range(len(tile_data)):
tile_data[i] = new_data[i]
wx.PostEvent(self.notify_window,
TileUpdatedEvent(obKey))
finally:
javabridge.detach()
def run(self):
javabridge.attach()
try:
while 1:
self.tile_collection.cv.acquire()
# If there are no objects in the queue then wait
while not self.tile_collection.loadq:
self.tile_collection.cv.wait()
if self._want_abort:
self.tile_collection.cv.release()
logging.info("%s aborted" % self.getName())
return
data = heappop(self.tile_collection.loadq)
obKey = data[1]
display_whole_image = data[2] # display whole image instead of object image
self.tile_collection.cv.release()
# wait until loading has completed before continuing
with self.tile_collection.load_lock:
# Make sure tile hasn't been deleted outside this thread
if not self.tile_collection.tileData.get(obKey, None):
continue
# Get the tile
new_data = imagetools.FetchTile(obKey, display_whole_image=display_whole_image)
if new_data is None:
# if fetching fails, leave the tile blank
continue
tile_data = self.tile_collection.tileData.get(obKey, None)
# Make sure tile hasn't been deleted outside this thread
if tile_data is not None:
# copy each channel
for i in range(len(tile_data)):
tile_data[i] = new_data[i]
wx.PostEvent(self.notify_window, TileUpdatedEvent(obKey))
finally:
javabridge.detach()
def fn():
from bioformats import load_image_url
import javabridge
from scipy.io.matlab.mio import loadmat
from cellprofiler.modules.loadimages import url2pathname
javabridge.attach()
with self.image_dict_lock:
generation = self.image_dict_generation
for k, v in well.iteritems():
sd = {}
with self.image_dict_lock:
if self.image_dict_generation > generation:
return
self.image_dict[k] = sd
for c, fd in enumerate(v):
if PlateData.D_CHANNEL in fd:
channel = fd[PlateData.D_CHANNEL]
else:
channel = str(c+1)
url = fd[PlateData.D_FILENAME]
try:
if url.lower().endswith(".mat"):
img = loadmat(
url2pathname(url),
struct_as_record=True)["Image"]
else:
img = load_image_url(url)
with self.image_dict_lock:
if self.image_dict_generation > generation:
return
sd[channel] = img
except:
traceback.print_exc()
pass
wx.CallAfter(self.update_figure)
javabridge.static_call(
"java/lang/System", "gc", "()V")
javabridge.detach()
def fn():
from bioformats import load_image_url
import javabridge
from scipy.io.matlab.mio import loadmat
from cellprofiler.modules.loadimages import url2pathname
javabridge.attach()
with self.image_dict_lock:
generation = self.image_dict_generation
for k, v in well.iteritems():
sd = {}
with self.image_dict_lock:
if self.image_dict_generation > generation:
return
self.image_dict[k] = sd
for c, fd in enumerate(v):
if PlateData.D_CHANNEL in fd:
channel = fd[PlateData.D_CHANNEL]
else:
channel = str(c + 1)
url = fd[PlateData.D_FILENAME]
try:
if url.lower().endswith(".mat"):
img = loadmat(
url2pathname(url),
struct_as_record=True)["Image"]
else:
img = load_image_url(url)
with self.image_dict_lock:
if self.image_dict_generation > generation:
return
sd[channel] = img
except:
traceback.print_exc()
pass
wx.CallAfter(self.update_figure)
javabridge.static_call(
"java/lang/System", "gc", "()V")
javabridge.detach()
def start(class_path=None,
bundled=True,
packages=False,
system_cp=False,
max_heap_size=None):
"""
Initializes the javabridge connection (starts up the JVM).
:param class_path: the additional classpath elements to add
:type class_path: str
:param bundled: whether to add jars from the "lib" directory
:type bundled: bool
:param packages: whether to add jars from Weka packages as well
:type packages: bool
:param system_cp: whether to add the system classpath as well
:type system_cp: bool
:param max_heap_size: the maximum heap size (-Xmx parameter, eg 512m or 4g)
:type max_heap_size: str
"""
global started
if not started is None:
return
# add user-defined jars first
if not class_path is None:
for cp in class_path:
javabridge.JARS.append(cp)
if bundled:
add_bundled_jars()
if packages:
add_weka_packages()
if system_cp:
add_system_classpath()
javabridge.start_vm(run_headless=True, max_heap_size=max_heap_size)
javabridge.attach()
started = True
def engine_requester():
J.attach()
while True:
try:
msg = J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
CPythonScriptEngine.engineRequestQueue.take();""")
if logger.level <= logging.INFO:
logger.info("Received engine request: %s", J.to_string(msg))
payload = J.get_collection_wrapper(
J.run_script("msg.payload", dict(msg=msg)))
if J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
msg.command==CPythonScriptEngine.EngineCommands.NEW_ENGINE;
""", dict(msg=msg)):
do_new_engine(payload)
elif J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
msg.command==CPythonScriptEngine.EngineCommands.CLOSE_SERVICE;
""", dict(msg=msg)):
logger.info("Exiting script service thread in response to "
"termination request")
break
else:
J.run_script(
"""importPackage(Packages.org.scijava.plugins.scripting.cpython);
var exception = new java.lang.RuntimeException(
java.lang.String.format('Unknown command: %s', msg.command.toString()));
var payload = new java.util.ArrayList();
payload.add(exception);
var response = new CPythonScriptEngine.Message(
CPythonScriptEngine.EngineCommands.EXCEPTION, payload);
CPythonScriptEngine.engineResponseQueue.put(response);
""")
except:
# To do: how to handle failure, probably from .take()
# Guessing that someone has managed to interrupt our thread
logger.warn("Exiting script service thread", exc_info=True)
J.detach()
def setUp(self):
self.env = javabridge.attach()
# <AK> added (temporary!!!)
if 0: #self.env.exception_occurred():
import sys
from jt import jni
print("\n@@ setUp", self.env.exception_occurred(), file=sys.stderr)
self.env.exception_describe()
print("@@ jni.Throwable.last", jni.Throwable.last, file=sys.stderr)
#self.env.exception_clear()
print("@@ /setUp", self.env.exception_occurred(), file=sys.stderr)
# </AK>
self.env.exception_clear() # <AK> added (temporary!!!)
def start(class_path=None, bundled=True, packages=False, system_cp=False, max_heap_size=None):
"""
Initializes the javabridge connection (starts up the JVM).
:param class_path: the additional classpath elements to add
:type class_path: str
:param bundled: whether to add jars from the "lib" directory
:type bundled: bool
:param packages: whether to add jars from Weka packages as well
:type packages: bool
:param system_cp: whether to add the system classpath as well
:type system_cp: bool
:param max_heap_size: the maximum heap size (-Xmx parameter, eg 512m or 4g)
:type max_heap_size: str
"""
global started
if not started is None:
return
# add user-defined jars first
if not class_path is None:
for cp in class_path:
javabridge.JARS.append(cp)
if bundled:
add_bundled_jars()
if packages:
add_weka_packages()
if system_cp:
add_system_classpath()
javabridge.start_vm(run_headless=True, max_heap_size=max_heap_size)
javabridge.attach()
started = True
def setUp(self):
javabridge.attach()
log4j.basic_config()
def run(my_integer = my_integer):
env = javabridge.attach()
self.assertEqual(my_integer.intValue(),my_value)
del my_integer
javabridge.detach()
def setUp(self):
self.env = javabridge.attach()
def interface(self,
start_signal,
image_set_start=1,
image_set_end=None,
overwrite=True):
'''Top-half thread for running an analysis. Sets up grouping for jobs,
deals with returned measurements, reports status periodically.
start_signal- signal this semaphore when jobs are ready.
image_set_start - beginning image set number to process
image_set_end - last image set number to process
overwrite - whether to recompute imagesets that already have data in initial_measurements.
'''
from javabridge import attach, detach
posted_analysis_started = False
acknowledged_thread_start = False
measurements = None
workspace = None
attach()
try:
# listen for pipeline events, and pass them upstream
self.pipeline.add_listener(lambda pipe, evt: self.post_event(evt))
initial_measurements = None
if self.output_path is None:
# Caller wants a temporary measurements file.
fd, filename = tempfile.mkstemp(
".h5", dir=cpprefs.get_temporary_directory())
try:
fd = os.fdopen(fd, "wb")
fd.write(self.initial_measurements_buf)
fd.close()
initial_measurements = cpmeas.Measurements(
filename=filename, mode="r")
measurements = cpmeas.Measurements(
image_set_start=None,
copy=initial_measurements,
mode="a")
finally:
if initial_measurements is not None:
initial_measurements.close()
os.unlink(filename)
else:
with open(self.output_path, "wb") as fd:
fd.write(self.initial_measurements_buf)
measurements = cpmeas.Measurements(image_set_start=None,
filename=self.output_path,
mode="a")
# The shared dicts are needed in jobserver()
self.shared_dicts = [m.get_dictionary() for m in self.pipeline.modules()]
workspace = cpw.Workspace(self.pipeline, None, None, None,
measurements, cpimage.ImageSetList())
if image_set_end is None:
image_set_end = measurements.get_image_numbers()[-1]
image_sets_to_process = filter(
lambda x: x >= image_set_start and x <= image_set_end,
measurements.get_image_numbers())
self.post_event(AnalysisStarted())
posted_analysis_started = True
# reset the status of every image set that needs to be processed
has_groups = measurements.has_groups()
if self.pipeline.requires_aggregation():
overwrite = True
if has_groups and not overwrite:
if not measurements.has_feature(cpmeas.IMAGE, self.STATUS):
overwrite = True
else:
group_status = {}
for image_number in measurements.get_image_numbers():
group_number = measurements[
cpmeas.IMAGE, cpmeas.GROUP_NUMBER, image_number]
status = measurements[cpmeas.IMAGE, self.STATUS,
image_number]
if status != self.STATUS_DONE:
group_status[group_number] = self.STATUS_UNPROCESSED
elif group_number not in group_status:
group_status[group_number] = self.STATUS_DONE
new_image_sets_to_process = []
for image_set_number in image_sets_to_process:
needs_reset = False
if (overwrite or
(not measurements.has_measurements(
cpmeas.IMAGE, self.STATUS, image_set_number)) or
(measurements[cpmeas.IMAGE, self.STATUS, image_set_number]
!= self.STATUS_DONE)):
needs_reset = True
elif has_groups:
group_number = measurements[
cpmeas.IMAGE, cpmeas.GROUP_NUMBER, image_set_number]
if group_status[group_number] != self.STATUS_DONE:
needs_reset = True
if needs_reset:
measurements[cpmeas.IMAGE, self.STATUS, image_set_number] = \
self.STATUS_UNPROCESSED
new_image_sets_to_process.append(image_set_number)
image_sets_to_process = new_image_sets_to_process
# Find image groups. These are written into measurements prior to
# analysis. Groups are processed as a single job.
if has_groups or self.pipeline.requires_aggregation():
worker_runs_post_group = True
job_groups = {}
for image_set_number in image_sets_to_process:
group_number = measurements[cpmeas.IMAGE,
cpmeas.GROUP_NUMBER,
image_set_number]
group_index = measurements[cpmeas.IMAGE,
cpmeas.GROUP_INDEX,
image_set_number]
job_groups[group_number] = job_groups.get(group_number, []) + [(group_index, image_set_number)]
job_groups = [[isn for _, isn in sorted(job_groups[group_number])]
for group_number in sorted(job_groups)]
else:
worker_runs_post_group = False # prepare_group will be run in worker, but post_group is below.
job_groups = [[image_set_number] for image_set_number in image_sets_to_process]
# XXX - check that any constructed groups are complete, i.e.,
# image_set_start and image_set_end shouldn't carve them up.
if not worker_runs_post_group:
# put the first job in the queue, then wait for the first image to
# finish (see the check of self.finish_queue below) to post the rest.
# This ensures that any shared data from the first imageset is
# available to later imagesets.
self.work_queue.put((job_groups[0],
worker_runs_post_group,
True))
waiting_for_first_imageset = True
del job_groups[0]
else:
waiting_for_first_imageset = False
for job in job_groups:
self.work_queue.put((job, worker_runs_post_group, False))
job_groups = []
start_signal.release()
acknowledged_thread_start = True
# We loop until every image is completed, or an outside event breaks the loop.
while not self.cancelled:
# gather measurements
while not self.received_measurements_queue.empty():
image_numbers, buf = self.received_measurements_queue.get()
image_numbers = [int(i) for i in image_numbers]
recd_measurements = cpmeas.load_measurements_from_buffer(buf)
self.copy_recieved_measurements(recd_measurements, measurements, image_numbers)
recd_measurements.close()
del recd_measurements
# check for jobs in progress
while not self.in_process_queue.empty():
image_set_numbers = self.in_process_queue.get()
for image_set_number in image_set_numbers:
measurements[cpmeas.IMAGE, self.STATUS, int(image_set_number)] = self.STATUS_IN_PROCESS
# check for finished jobs that haven't returned measurements, yet
while not self.finished_queue.empty():
finished_req = self.finished_queue.get()
measurements[
cpmeas.IMAGE, self.STATUS, int(finished_req.image_set_number)] = self.STATUS_FINISHED_WAITING
if waiting_for_first_imageset:
assert isinstance(finished_req,
ImageSetSuccessWithDictionary)
self.shared_dicts = finished_req.shared_dicts
waiting_for_first_imageset = False
assert len(self.shared_dicts) == len(self.pipeline.modules())
# if we had jobs waiting for the first image set to finish,
# queue them now that the shared state is available.
for job in job_groups:
self.work_queue.put((job, worker_runs_post_group, False))
finished_req.reply(Ack())
# check progress and report
counts = collections.Counter(measurements[cpmeas.IMAGE, self.STATUS, image_set_number]
for image_set_number in image_sets_to_process)
self.post_event(AnalysisProgress(counts))
# Are we finished?
if counts[self.STATUS_DONE] == len(image_sets_to_process):
last_image_number = measurements.get_image_numbers()[-1]
measurements.image_set_number = last_image_number
if not worker_runs_post_group:
self.pipeline.post_group(workspace, {})
workspace = cpw.Workspace(self.pipeline,
None, None, None,
measurements, None, None)
workspace.post_run_display_handler = \
self.post_run_display_handler
self.pipeline.post_run(workspace)
break
measurements.flush()
# not done, wait for more work
with self.interface_work_cv:
while (self.paused or
((not self.cancelled) and
self.in_process_queue.empty() and
self.finished_queue.empty() and
self.received_measurements_queue.empty())):
self.interface_work_cv.wait() # wait for a change of status or work to arrive
finally:
detach()
# Note - the measurements file is owned by the queue consumer
# after this post_event.
#
if not acknowledged_thread_start:
start_signal.release()
if posted_analysis_started:
was_cancelled = self.cancelled
self.post_event(AnalysisFinished(measurements, was_cancelled))
self.stop_workers()
self.analysis_id = False # this will cause the jobserver thread to exit
def setUp(self):
from cellprofiler.modules.tests import example_images_directory
self.root_dir = example_images_directory()
J.attach()
def __init__(self, df, penaltydiscount = 4, depth = 3, faithfulness = True, numofthreads = 2, verbose = False, java_max_heap_size = None):
tetrad_libdir = os.path.join(os.path.dirname(__file__), 'lib')
for l in glob.glob(tetrad_libdir + os.sep + "*.jar"):
javabridge.JARS.append(str(l))
javabridge.start_vm(run_headless=True, max_heap_size = java_max_heap_size)
javabridge.attach()
tetradData = None
if(len(df.index)*df.columns.size <= 1500):
node_list = javabridge.JClassWrapper('java.util.ArrayList')()
for col in df.columns:
nodi = javabridge.JClassWrapper('edu.cmu.tetrad.data.ContinuousVariable')(col)
node_list.add(nodi)
dataBox = javabridge.JClassWrapper('edu.cmu.tetrad.data.DoubleDataBox')(len(df.index),df.columns.size)
for row in df.index:
for col in range(0,df.columns.size):
value = javabridge.JClassWrapper('java.lang.Double')(df.ix[row][col])
dataBox.set(row,col,value)
tetradData = javabridge.JClassWrapper('edu.cmu.tetrad.data.BoxDataSet')(dataBox, node_list)
else:
#Generate random name
temp_data_file = ''.join(random.choice(string.lowercase) for i in range(10)) + '.csv'
temp_data_path = os.path.join(tempfile.gettempdir(), temp_data_file)
df.to_csv(temp_data_path, sep = '\t', index = False)
excludeVar = javabridge.JClassWrapper('java.util.HashSet')()
excludeVar.add('MULT')
# Read in method 1
f = javabridge.JClassWrapper('java.nio.file.Paths').get(temp_data_path)
dataReader = javabridge.JClassWrapper('edu.cmu.tetrad.io.TabularContinuousDataReader')(f,'\t')
tetradData = dataReader.readInData(excludeVar)
# Read in method 2 -- Depreciated
# f = javabridge.JClassWrapper('java.io.File')(temp_data_path)
# tetradData = javabridge.JClassWrapper('edu.cmu.tetrad.data.BigDataSetUtility').readInContinuousData(f,'\t',excludeVar)
os.remove(temp_data_path)
tetradData = javabridge.JClassWrapper('edu.cmu.tetrad.data.CovarianceMatrixOnTheFly')(tetradData)
score = javabridge.JClassWrapper('edu.cmu.tetrad.search.SemBicScore')(tetradData)
score.setPenaltyDiscount(penaltydiscount) # set to 2 if variable# <= 50 otherwise set it to 4
fgs = javabridge.JClassWrapper('edu.cmu.tetrad.search.Fgs')(score)
fgs.setDepth(depth)#-1
fgs.setNumPatternsToStore(0)
fgs.setFaithfulnessAssumed(faithfulness)
fgs.setParallelism(numofthreads)
fgs.setVerbose(verbose)
tetradGraph = fgs.search()
graph = pydot.Dot(graph_type='digraph')
n = tetradGraph.getNodeNames().toString()
n = n[1:len(n)-1]
n = n.split(",")
nodes = []
for i in range(0,len(n)):
node = n[i]
n[i] = node.strip()
nodes.append(pydot.Node(n[i]))
graph.add_node(nodes[i])
self.nodes = n
e = tetradGraph.getEdges().toString()
e = e[1:len(e)-1]
e = e.split(",")
for i in range(0,len(e)):
e[i] = e[i].strip()
token = e[i].split(" ")
if(len(token) == 3):
src = token[0]
arc = token[1]
dst = token[2]
if(pycausal.isNodeExisting(n,src) and pycausal.isNodeExisting(n,dst)):
edge = pydot.Edge(nodes[n.index(src)],nodes[n.index(dst)])
if(arc == "---"):
edge.set_arrowhead("none")
graph.add_edge(edge)
self.edges = e
javabridge.detach()
javabridge.kill_vm()
self.graph = graph
def setUp(self):
J.attach()
def __init__(self, df, structurePrior = 1.0, samplePrior = 1.0, depth = 3, faithfulness = True, numofthreads = 2, verbose = False, java_max_heap_size = None):
tetrad_libdir = os.path.join(os.path.dirname(__file__), 'lib')
for l in glob.glob(tetrad_libdir + os.sep + "*.jar"):
javabridge.JARS.append(str(l))
javabridge.start_vm(run_headless=True, max_heap_size = java_max_heap_size)
javabridge.attach()
score = None
# if(len(df.index)*df.columns.size <= 1500):
dataBox = javabridge.JClassWrapper('edu.cmu.tetrad.data.VerticalIntDataBox')(len(df.index),df.columns.size)
node_list = javabridge.JClassWrapper('java.util.ArrayList')()
col_no = 0
for col in df.columns:
cat_array = sorted(set(df[col]))
cat_list = javabridge.JClassWrapper('java.util.ArrayList')()
for cat in cat_array:
catname = javabridge.JClassWrapper('java.lang.String')(cat)
cat_list.add(catname)
nodname = javabridge.JClassWrapper('java.lang.String')(col)
nodi = javabridge.JClassWrapper('edu.cmu.tetrad.data.DiscreteVariable')(nodname,cat_list)
node_list.add(nodi)
for row in df.index:
value = javabridge.JClassWrapper('java.lang.Integer')(cat_array.index(df.ix[row][col_no]))
dataBox.set(row,col_no,value)
col_no = col_no + 1
tetradData = javabridge.JClassWrapper('edu.cmu.tetrad.data.BoxDataSet')(dataBox, node_list)
# else:
# Generate random name
# temp_data_file = ''.join(random.choice(string.lowercase) for i in range(10)) + '.csv'
# temp_data_path = os.path.join(tempfile.gettempdir(), temp_data_file)
# df.to_csv(temp_data_path, sep = "\t", index = False)
# excludeVar = javabridge.JWrapper(javabridge.make_instance("java/util/HashSet","()V"))
# excludeVar.add("MULT")
# Read in method 1
# f = javabridge.static_call("java/nio/file/Paths","get","(Ljava/lang/String;)V",temp_data_path)
# dataReader = javabridge.make_instance("edu/cmu/tetrad/io/TabularContinuousDataReader","(Ljava/nio/file/Path;C)V",f,"\t")
# dataReader = javabridge.JWrapper(dataReader)
# tetradData = dataReader.readInData(excludeVar)
# Read in method 2 -- Depreciated
# f = javabridge.make_instance("java/io/File", "(Ljava/lang/String;)V",temp_data_path)
# tetradData = javabridge.static_call("edu/cmu/tetrad/data/BigDataSetUtility", "readInDiscreteData", "(Ljava/io/File;CLjava/util/Set;)Ledu/cmu/tetrad/data/DataSet;",f,"\t",excludeVar)
# os.remove(temp_data_path)
score = javabridge.JClassWrapper('edu.cmu.tetrad.search.BDeuScore')(tetradData)
score.setStructurePrior(structurePrior)
score.setSamplePrior(samplePrior)
fgs = javabridge.JClassWrapper('edu.cmu.tetrad.search.Fgs')(score)
fgs.setDepth(depth)
fgs.setNumPatternsToStore(0)
fgs.setFaithfulnessAssumed(faithfulness)
fgs.setParallelism(numofthreads)
fgs.setVerbose(verbose)
tetradGraph = fgs.search()
graph = pydot.Dot(graph_type='digraph')
n = tetradGraph.getNodeNames().toString()
n = n[1:len(n)-1]
n = n.split(",")
nodes = []
for i in range(0,len(n)):
node = n[i]
n[i] = node.strip()
nodes.append(pydot.Node(n[i]))
graph.add_node(nodes[i])
self.nodes = n
e = tetradGraph.getEdges().toString()
e = e[1:len(e)-1]
e = e.split(",")
for i in range(0,len(e)):
e[i] = e[i].strip()
token = e[i].split(" ")
if(len(token) == 3):
src = token[0]
arc = token[1]
dst = token[2]
if(pycausal.isNodeExisting(n,src) and pycausal.isNodeExisting(n,dst)):
edge = pydot.Edge(nodes[n.index(src)],nodes[n.index(dst)])
if(arc == "---"):
edge.set_arrowhead("none")
graph.add_edge(edge)
self.edges = e
javabridge.detach()
javabridge.kill_vm()
self.graph = graph
def run(self): # Attach JVM javabridge.attach() # Debug print "Classifier" print self.classifier print "Params" print self.parameters print "Model Params" print self.modelParams # Get data for testing and learning learnerData = self.retrieveData(self.questionID, "learner") testData = self.retrieveData(self.questionID, 'test') masterData = self.retrieveData(self.questionID, 'all') masterData = self.addNominals(masterData) # Check if there is enough correct data to run if (learnerData.num_instances < 1 or testData.num_instances < 1): self.status = self.config.NOT_ENOUGH_DATA return False # If this is a prediction and there is a valid patient, change masterData header patientObj = self.buildPatientObject() patientInstance = None if ((patientObj is not None) and (self.predict == 1)): masterData = self.addPatientNominals(patientObj, masterData) patientInstance = self.createPatientInstance(patientObj, masterData) masterData.add_instance(patientInstance) elif (patientObj is None) and (self.predict == 1): print 'No patient defined for prediction. Exiting' return True # Fix dataset headers up to match and fix instances to match headers masterData.delete() learner = masterData.copy_instances(masterData, 0, 0) test = masterData.copy_instances(masterData, 0, 0) self.addInstancesToDataset(learnerData, learner) self.addInstancesToDataset(testData, test) # Comparison of data for testing purposes # print 'learnerData' # print learnerData # print 'learner' # print learner # print 'testData' # print testData # print 'test' # print test # pdb.set_trace() # Instantiate classifier self.cls = Classifier(classname=self.classifier, options=self.parameters) # Run classifier self.cls.build_classifier(learner) # for index, inst in enumerate(learnerData): # prediction = self.cls.classify_instance(inst) # distribution = self.cls.distribution_for_instance(inst) # Test classifier evl = Evaluation(learner) evl.test_model(self.cls, test) # Store information about matrix self.acc = evl.percent_correct self.val = None # Convert numpy array into simple array confusionMatrix = [] confusionMatrix.append([evl.confusion_matrix[0][0], evl.confusion_matrix[0][1]]) confusionMatrix.append([evl.confusion_matrix[1][0], evl.confusion_matrix[1][1]]) # Convert matrix into json format self.matrix = json.dumps(confusionMatrix) # print 'Classifier: ', self.classifier # print 'ID: ', self.questionID # print 'ACC: ', self.acc # print(evl.summary()) # If this is a prediction... make the prediction if ((patientObj is not None) and (self.predict == 1)): masterData.add_instance(patientInstance) print "Running prediction on patient: " print masterData.get_instance(0) self.prediction = self.cls.classify_instance(masterData.get_instance(0)) #self.uploadPrediction() # Temporarily store file to serialize to fileName = str(self.questionID) + self.algorithm + ".model" serialization.write(fileName, self.cls) # Open that file and store it self.model = None with open(fileName, 'rb') as f: self.model = f.read() # Remove temporary file os.remove(fileName) # Set status to awaiting feedback self.status = self.config.AWAITING_FEEDBACK_STATUS return True
def setUp(self):
J.attach()
log4j.basic_config()
def process_meta(func, df, schema_name, overwrite=False, **kwargs):
"""Extract metadata from a Datafile using a provided function and save the
outputs as DatafileParameters.
Parameters
----------
func: Function
Function to extract metadata from a file. Function must have
input_file_path as an argument e.g.:
def meta_proc(input_file_path, **kwargs):
...
It must return a dict containing ParameterNames as keys and the
Parameters to be saved as values. Parameters (values) can be singular
strings/numerics or a list of strings/numeric. If it's a list, each
element will be saved as a new DatafileParameter.
df: tardis.tardis_portal.models.Datafile
Datafile instance to process.
schema_name: str
Names of schema which describes ParameterNames
add: boolean (default: False)
Specifies whether or not to add to an existing Parameterset for this
Datafile rather that overwriting or exiting.
overwrite: boolean (default: False)
Specifies whether to overwrite any exisiting parametersets for
this datafile.
Returns
-------
None
"""
if acquire_datafile_lock(df.id):
# Need to start a JVM in each thread
check_and_start_jvm()
try:
javabridge.attach()
log4j.basic_config()
schema = Schema.objects.get(namespace__exact=schema_name)
if DatafileParameterSet.objects\
.filter(schema=schema, datafile=df).exists():
if overwrite:
psets = DatafileParameterSet.objects.get(schema=schema,
datafile=df)
logger.warning("Overwriting parametersets for %s"
% df.filename)
[delete_old_parameterset(ps) for ps in psets]
else:
logger.warning("Parametersets for %s already exist."
% df.filename)
return
dfo = DataFileObject.objects.filter(datafile__id=df.id,
verified=True).first()
input_file_path = dfo.get_full_path()
logger.debug("Processing file: %s" % input_file_path)
metadata_params = func(input_file_path, **kwargs)
if not metadata_params:
logger.debug("No metadata to save")
return
for sm in metadata_params:
ps = DatafileParameterSet(schema=schema, datafile=df)
ps.save()
logger.debug("Saving parameters for: %s" % input_file_path)
save_parameters(schema, ps, sm)
except Exception, e:
logger.debug(e)
finally:
def Start(self):
'''Initialize CPA
'''
'''List of tables created by the user during this session'''
self.user_tables = []
# splashscreen
splashimage = cpa.icons.cpa_splash.ConvertToBitmap()
# If the splash image has alpha, it shows up transparently on
# windows, so we blend it into a white background.
splashbitmap = wx.EmptyBitmapRGBA(splashimage.GetWidth(),
splashimage.GetHeight(),
255, 255, 255, 255)
dc = wx.MemoryDC()
dc.SelectObject(splashbitmap)
dc.DrawBitmap(splashimage, 0, 0)
dc.Destroy() # necessary to avoid a crash in splashscreen
splash = wx.SplashScreen(splashbitmap, wx.SPLASH_CENTRE_ON_SCREEN |
wx.SPLASH_TIMEOUT, 2000, None, -1)
self.splash = splash
p = Properties.getInstance()
if not p.is_initialized():
from cpa.guiutils import show_load_dialog
splash.Destroy()
if not show_load_dialog():
splash.Destroy()
example_link_address = 'cellprofiler.org'
dlg = wx.MessageDialog(None, 'CellProfiler Analyst requires a properties file. Download an example at %s' % (
example_link_address), 'Properties file required', wx.OK)
response = dlg.ShowModal()
logging.error('CellProfiler Analyst requires a properties file. Exiting.')
return False
self.frame = MainGUI(p, None, size=(1000,-1))
db = DBConnect.getInstance()
# Black magic: Bus errors occur on Mac OS X if we wait until
# the JVM or the wx event look has started to connect. But it
# has to be done after we have read the properties file. So we
# do it here.
db.connect()
db.register_gui_parent(self.frame)
# The JVM has to be started after db.connect(), otherwise bus errors
# occur on Mac OS X.
javabridge.start_vm(class_path=bioformats.JARS, run_headless=True)
# removes the log4j warnings
javabridge.attach()
# TODO: check for updates
try:
if __version__ != -1:
import cpa.util.check_for_updates as cfu
cfu.check_for_updates('http://cellprofiler.org/updates/CPA.html',
max(__version__, cpa.cpaprefs.get_skip_version()),
new_version_cb,
user_agent='CPAnalyst/%s'%(__version__))
except ImportError:
logging.warn("CPA was unable to check for updates. Could not import cpa.util.check_for_updates.")
self.frame.Show() # Show frame
return True
def run(my_integer=my_integer):
env = javabridge.attach()
self.assertEqual(my_integer.intValue(), my_value)
del my_integer
javabridge.detach()
def setUp(self):
J.start_vm(class_path=bioformats.JARS)
J.attach()
bioformats.init_logger()
def setUp(self):
self.env = javabridge.attach()
def __init__(self, df, penaltydiscount = 4, depth = 3, faithfulness = True, verbose = False, java_max_heap_size = None):
tetrad_libdir = os.path.join(os.path.dirname(__file__), 'lib')
# print 'tetrad_libdir: %s' % tetrad_libdir
for l in glob.glob(tetrad_libdir + os.sep + "*.jar"):
javabridge.JARS.append(str(l))
javabridge.start_vm(run_headless=True, max_heap_size = java_max_heap_size)
javabridge.attach()
tetradData = None
if(len(df.index)*df.columns.size <= 1500):
node_list = javabridge.JWrapper(javabridge.make_instance("java/util/ArrayList", "()V"))
for col in df.columns:
nodname = javabridge.make_instance("java/lang/String", "(Ljava/lang/String;)V",col)
nodi = javabridge.make_instance("edu/cmu/tetrad/graph/GraphNode", "(Ljava/lang/String;)V",nodname)
node_list.add(nodi)
tetradMatrix = javabridge.JWrapper(javabridge.make_instance("edu/cmu/tetrad/util/TetradMatrix","(II)V",
len(df.index),df.columns.size))
for row in df.index:
for col in range(0,df.columns.size):
tetradMatrix.set(row,col,df.ix[row][col])
tetradData = javabridge.static_call("edu/cmu/tetrad/data/ColtDataSet","makeContinuousData",
"(Ljava/util/List;Ledu/cmu/tetrad/util/TetradMatrix;)Ledu/cmu/tetrad/data/DataSet;",
node_list,tetradMatrix)
else:
#Generate random name
temp_data_file = ''.join(random.choice(string.lowercase) for i in range(10)) + '.csv'
temp_data_path = os.path.join(tempfile.gettempdir(), temp_data_file)
df.to_csv(temp_data_path, sep = "\t")
f = javabridge.make_instance("java/io/File", "(Ljava/lang/String;)V",temp_data_path)
excludeVar = javabridge.JWrapper(javabridge.make_instance("java/util/HashSet","()V"))
excludeVar.add("MULT")
tetradData = javabridge.static_call("edu/cmu/tetrad/data/BigDataSetUtility","readInContinuousData","(Ljava/io/File;CLjava/util/Set;)Ledu/cmu/tetrad/data/DataSet;",f,"\t",excludeVar)
os.remove(temp_data_path)
fgs = javabridge.make_instance("edu/cmu/tetrad/search/Fgs","(Ledu/cmu/tetrad/data/DataSet;)V",tetradData)
fgs = javabridge.JWrapper(fgs)
fgs.setPenaltyDiscount(penaltydiscount)# set to 2 if variable# <= 50 otherwise set it to 4
fgs.setDepth(depth)#-1
fgs.setNumPatternsToStore(0)
fgs.setFaithfulnessAssumed(faithfulness)
fgs.setVerbose(verbose)
tetradGraph = fgs.search()
graph = pydot.Dot(graph_type='digraph')
n = tetradGraph.getNodeNames().toString()
n = n[1:len(n)-1]
n = n.split(",")
nodes = []
for i in range(0,len(n)):
node = n[i]
n[i] = node.strip()
nodes.append(pydot.Node(n[i]))
graph.add_node(nodes[i])
self.nodes = n
e = tetradGraph.getEdges().toString()
e = e[1:len(e)-1]
e = e.split(",")
for i in range(0,len(e)):
e[i] = e[i].strip()
token = e[i].split(" ")
if(len(token) == 3):
src = token[0]
arc = token[1]
dst = token[2]
if(pycausal.isNodeExisting(n,src) and pycausal.isNodeExisting(n,dst)):
edge = pydot.Edge(nodes[n.index(src)],nodes[n.index(dst)])
if(arc == "---"):
edge.set_arrowhead("none")
graph.add_edge(edge)
self.edges = e
javabridge.detach()
javabridge.kill_vm()
self.graph = graph
# filename = dlg.Path
# else:
# app.Exit()
# sys.exit()
filename = '/Users/afraser/Desktop/cpa_example/images/AS_09125_050116000001_A01f00d0.png'
filename = '/Users/afraser/Desktop/wedding/header.jpg'
out_file = '/Users/afraser/Desktop/test_output.avi'
try:
os.remove(out_file)
print('previous output file deleted')
except:
print('no output file to delete')
env = jutil.attach()
ImageReader = make_image_reader_class()
ChannelSeparator = make_reader_wrapper_class("loci/formats/ChannelSeparator")
FormatTools = make_format_tools_class()
# writer testing
ImageWriter = make_image_writer_class()
writer = ImageWriter()
w = 400
h = 400
c = 3
z = 1
t = 4
images = []
for tt in range(t):
# filename = dlg.Path
# else:
# app.Exit()
# sys.exit()
filename = '/Users/afraser/Desktop/cpa_example/images/AS_09125_050116000001_A01f00d0.png'
filename = '/Users/afraser/Desktop/wedding/header.jpg'
out_file = '/Users/afraser/Desktop/test_output.avi'
try:
os.remove(out_file)
print('previous output file deleted')
except:
print('no output file to delete')
env = jutil.attach()
ImageReader = make_image_reader_class()
ChannelSeparator = make_reader_wrapper_class(
"loci/formats/ChannelSeparator")
FormatTools = make_format_tools_class()
# writer testing
ImageWriter = make_image_writer_class()
writer = ImageWriter()
w = 400
h = 400
c = 3
z = 1
t = 4
images = []
def setUp(self):
from cellprofiler.modules.tests import example_images_directory
self.root_dir = example_images_directory()
J.attach()
@app.route("/start", methods=["GET"])
def init():
start()
return "starting CPA"
##
## Actually setup the Api resource routing here
##
api.add_resource(helloworld, "/")
# api.add_resource(TrainingSet, '/')
api.add_resource(getImagePaths, "/images")
api.add_resource(getBase64, "/base64")
api.add_resource(getAll, "/all")
# Server Main Function
if __name__ == "__main__":
# app.debug = True
# app.run(host='0.0.0.0', port=5000) #Public IP
start()
# Start the virtual machine
javabridge.start_vm(class_path=bioformats.JARS, run_headless=True)
javabridge.attach()
javabridge.activate_awt()
# Calculate the Training DataSet and store it
calculateTrainingSetJSON(base64=True)