def testPyramidConcurrentAccess(self): """ See ticket:11709 """ all_context = {"omero.group": "-1"} from omero.util import concurrency pix = self.missing_pyramid(self.root) rps = self.root.sf.createRawPixelsStore(all_context) try: # First execution should certainly fail try: rps.setPixelsId(pix.id.val, True, all_context) assert False, "Should throw!" except omero.MissingPyramidException, mpm: assert pix.id.val == mpm.pixelsID # Eventually, however, it should be generated i = 10 success = False while i > 0 and not success: try: rps.setPixelsId(pix.id.val, True, all_context) success = True except omero.MissingPyramidException, mpm: assert pix.id.val == mpm.pixelsID backOff = mpm.backOff/1000 event = concurrency.get_event("testRomio") event.wait(backOff) # seconds i -= 1
def testPyramidConcurrentAccess(self, tmpdir): """ See ticket:11709 """ all_context = {"omero.group": "-1"} from omero.util import concurrency pix = self.missing_pyramid() rps = self.sf.createRawPixelsStore(all_context) try: # First execution should certainly fail try: rps.setPixelsId(long(pix), True, all_context) assert False, "Should throw!" except omero.MissingPyramidException, mpm: assert long(pix) == mpm.pixelsID # Eventually, however, it should be generated i = 10 success = False while i > 0 and not success: try: rps.setPixelsId(long(pix), True, all_context) success = True except omero.MissingPyramidException, mpm: assert long(pix) == mpm.pixelsID backOff = mpm.backOff / 1000 event = concurrency.get_event("testRomio") event.wait(backOff) # seconds i -= 1
def testRomioToPyramidWithNegOne(self): """ Here we try the above but pass omero.group:-1 to see if we can cause an exception. """ all_context = {"omero.group":"-1"} from omero.util import concurrency pix = self.missing_pyramid(self.root) rps = self.root.sf.createRawPixelsStore(all_context) try: # First execution should certainly fail try: rps.setPixelsId(pix.id.val, True, all_context) fail("Should throw!") except omero.MissingPyramidException, mpm: self.assertEquals(pix.id.val, mpm.pixelsID) # Eventually, however, it should be generated i = 10 success = False while i > 0 and not success: try: rps.setPixelsId(pix.id.val, True, all_context) success = True except omero.MissingPyramidException, mpm: self.assertEquals(pix.id.val, mpm.pixelsID) backOff = mpm.backOff/1000 event = concurrency.get_event("testRomio") event.wait(backOff) # seconds i -=1
def testRomioToPyramid(self): """ Here we create a pixels that is not big, then modify its metadata so that it IS big, in order to trick the service into throwing us a MissingPyramidException """ from omero.util import concurrency pix = self.missing_pyramid(self.root) rps = self.root.sf.createRawPixelsStore() try: # First execution should certainly fail try: rps.setPixelsId(pix.id.val, True) fail("Should throw!") except omero.MissingPyramidException, mpm: self.assertEquals(pix.id.val, mpm.pixelsID) # Eventually, however, it should be generated i = 10 success = False while i > 0 and not success: try: rps.setPixelsId(pix.id.val, True) success = True except omero.MissingPyramidException, mpm: self.assertEquals(pix.id.val, mpm.pixelsID) backOff = mpm.backOff/1000 event = concurrency.get_event("testRomio") event.wait(backOff) # seconds i -=1
def keepalive(self, args): import threading from omero.util.concurrency import get_event as get_event class T(threading.Thread): def run(self): while self.client: try: self.client.sf.keepAlive(None) self.event.wait(args.frequency) except Exception as e: self.err("Keep alive failed: %s" % str(e)) return t = T() t.client = self.ctx.conn(args) t.err = self.ctx.err t.event = get_event(name="keepalive") t.start() try: self.ctx.out("Running keep alive every %s seconds" % args.frequency) self.ctx.input("Press enter to cancel.") finally: t.client = None t.event.set()
def testRomioToPyramid(self, tmpdir): """ Here we create a pixels that is not big, then modify its metadata so that it IS big, in order to trick the service into throwing us a MissingPyramidException """ from omero.util import concurrency pix = self.missing_pyramid() rps = self.sf.createRawPixelsStore() try: # First execution should certainly fail try: rps.setPixelsId(long(pix), True) assert False, "Should throw!" except omero.MissingPyramidException, mpm: assert long(pix) == mpm.pixelsID # Eventually, however, it should be generated i = 10 success = False while i > 0 and not success: try: rps.setPixelsId(long(pix), True) success = True except omero.MissingPyramidException, mpm: assert long(pix) == mpm.pixelsID backOff = mpm.backOff / 1000 event = concurrency.get_event("testRomio") event.wait(backOff) # seconds i -= 1
def __init__(self, controls = None, params = None, prog = sys.argv[0]): self.controls = controls if self.controls is None: self.controls = {} self.params = params if self.params is None: self.params = {} self.event = get_event(name="CLI") self.dir = OMERODIR self.isdebug = DEBUG # This usage will go away and default will be False self.topics = {"debug":""" debug options for developers: The value to the debug argument is a comma-separated list of commands: * 'debug' prints at the "debug" level. Similar to setting DEBUG=1 in the environment. * 'trace' runs the command with tracing enabled. * 'profile' runs the command with profiling enabled. Only one of "trace" and "profile" can be chosen. Example: bin/omero --debug=debug,trace admin start # Debugs at level 1 and prints tracing bin/omero -d1 admin start # Debugs at level 1 bin/omero -dp admin start # Prints profiling bin/omero -dt,p admin start # Fails!; can't print tracing and profiling together bin/omero -d0 admin start # Disables debugging """} self.parser = Parser(prog = prog, description = OMERODOC) self.subparsers = self.parser_init(self.parser)
def test2RomioToPyramidWithNegOne(self, tmpdir): """ Here we try the above but pass omero.group:-1 to see if we can cause an exception. """ all_context = {"omero.group": "-1"} from omero.util import concurrency pix = self.missing_pyramid() rps = self.sf.createRawPixelsStore(all_context) try: # First execution should certainly fail try: rps.setPixelsId(int(pix), True, all_context) assert False, "Should throw!" except omero.MissingPyramidException as mpm: assert int(pix) == mpm.pixelsID # Eventually, however, it should be generated i = 10 success = False while i > 0 and not success: try: rps.setPixelsId(int(pix), True, all_context) success = True except omero.MissingPyramidException as mpm: assert int(pix) == mpm.pixelsID backOff = old_div(mpm.backOff, 1000) event = concurrency.get_event("testRomio") event.wait(backOff) # seconds i -= 1 assert success finally: rps.close()
def __init__(self, client, handle, test): super(TestCB, self).__init__(client, handle) self.t_test = test self.t_lock = threading.RLock() self.t_steps = 0 self.t_finished = 0 self.t_event = get_event("TestCB")
def __init__(self, dir, communicator, getUsedFiles=as_dictionary, ctx=None, worker_wait=60, worker_count=1, worker_batch=10): """ Intialise the instance variables. """ self.log = logging.getLogger("fsclient." + __name__) self.communicator = communicator self.master = None #: Reference back to FSServer. self.serverProxy = None self.selfProxy = None self.dropBoxDir = dir self.host = "" self.port = 0 self.dirImportWait = 0 self.throttleImport = 5 self.timeToLive = 0 self.timeToIdle = 0 self.readers = "" self.importArgs = "" #: Id self.id = '' # Overriding methods to allow for simpler testing self.getUsedFiles = perf(getUsedFiles) # Threading primitives self.worker_wait = worker_wait self.worker_count = worker_count self.worker_batch = worker_batch self.event = get_event() self.queue = Queue.Queue(0) self.state = MonitorState(self.event) self.resources = Resources(stop_event=self.event) if ctx: # Primarily used for testing self.ctx = ctx else: self.ctx = ServerContext( server_id="DropBox", communicator=communicator, stop_event=self.event) self.resources.add(self.ctx) self.workers = [ MonitorWorker( worker_wait, worker_batch, self.event, self.queue, self.callback) for x in range(worker_count)] for worker in self.workers: worker.start() self.eventRecord("Directory", self.dropBoxDir)
'propagate': True }, '': { 'handlers': ['default'], 'level': 'DEBUG', 'propagate': True } } } # Load custom settings from etc/grid/config.xml # Tue 2 Nov 2010 11:03:18 GMT -- ticket:3228 from omero.util.concurrency import get_event CONFIG_XML = os.path.join(OMERO_HOME, 'etc', 'grid', 'config.xml') count = 10 event = get_event("websettings") while True: try: CUSTOM_SETTINGS = dict() if os.path.exists(CONFIG_XML): CONFIG_XML = omero.config.ConfigXml(CONFIG_XML, read_only=True) CUSTOM_SETTINGS = CONFIG_XML.as_map() CONFIG_XML.close() break except portalocker.LockException: # logger.error("Exception while loading configuration retrying...", # exc_info=True) exctype, value = sys.exc_info()[:2] count -= 1 if not count:
def keepalive(self, args): import threading from omero.util.concurrency import get_event as get_event class T(threading.Thread): def run(self): while self.client: try: self.client.sf.keepAlive(None) self.event.wait(args.frequency) except Exception, e: self.err("Keep alive failed: %s" % str(e)) return t = T() t.client = self.ctx.conn(args) t.event = get_event(name="keepalive") t.start() try: self.ctx.out("Running keep alive every %s seconds" % args.frequency) self.ctx.input("Press enter to cancel.") finally: t.client = None t.event.set() def file(self, args): """Return the file associated with the current active session""" store = self.store(args) srv, usr, uuid, port = store.get_current() if srv and usr and uuid: self.ctx.out(str(store.dir / srv / usr / uuid))
'propagate': True }, '': { 'handlers': ['default'], 'level': 'DEBUG', 'propagate': True } } } # Load custom settings from etc/grid/config.xml # Tue 2 Nov 2010 11:03:18 GMT -- ticket:3228 from omero.util.concurrency import get_event CONFIG_XML = os.path.join(OMERO_HOME, 'etc', 'grid', 'config.xml') count = 10 event = get_event("websettings") while True: try: CONFIG_XML = omero.config.ConfigXml(CONFIG_XML) CUSTOM_SETTINGS = CONFIG_XML.as_map() CONFIG_XML.close() break except LockException: #logger.error("Exception while loading configuration retrying...", exc_info=True) exctype, value = sys.exc_info()[:2] count -= 1 if not count: raise exctype, value else: event.wait(1) # Wait a total of 10 seconds
success = False while i > 0 and not success: try: rps.setPixelsId(pix.id.val, True, all_context) success = True except omero.MissingPyramidException, mpm: assert pix.id.val == mpm.pixelsID backOff = mpm.backOff/1000 event = concurrency.get_event("testRomio") event.wait(backOff) # seconds i -= 1 assert success # Once it's generated, we should be able to concurrencly # access the file without exceptions event = concurrency.get_event("concurrenct_pyramids") root_sf = self.root.sf class T(threading.Thread): def run(self): self.success = 0 self.failure = 0 while not event.isSet() and self.success < 10: self.rps = root_sf.createRawPixelsStore(all_context) try: self.rps.setPixelsId(pix.id.val, True, all_context) self.success += 1 except: self.failure += 1 raise finally: self.rps.close()
def testThumbnailVersion(self, meth): assert meth in ("one", "set") i64 = rint(64) pix = self.missing_pyramid() q = ("select tb from Thumbnail tb " "where tb.pixels.id = %s " "order by tb.id desc ") q = q % pix p = ParametersI().page(0, 1) get = lambda: self.query.findByQuery(q, p) # Before anything has been called, there # should be no thumbnail assert not get() # At this stage, there should still be no # thumbnail tb = self.client.sf.createThumbnailStore() if meth == "one": tb.setPixelsId(long(pix)) tb.resetDefaults() assert not tb.thumbnailExists(i64, i64) assert tb.isInProgress() # As soon as it's requested, it should have a -1 # version to mark pyramid creation as ongoing. if meth == "one": before = tb.getThumbnail(i64, i64) assert not tb.thumbnailExists(i64, i64) assert tb.isInProgress() elif meth == "set": before = tb.getThumbnailSet(i64, i64, [long(pix)]) before = before[long(pix)] assert get().version.val == -1 # Now we wait until the pyramid has been created # and test that a proper version has been set. event = get_event("test_thumbs") secs = 20 rps = self.client.sf.createRawPixelsStore() for x in range(secs): try: rps.setPixelsId(long(pix), True) event = None break except MissingPyramidException: event.wait(1) if event: assert "Pyramid was not generated %ss" % secs if meth == "one": # Re-load the thumbnail store now that # the pyramid is generated. tb.close() tb = self.client.sf.createThumbnailStore() if not tb.setPixelsId(long(pix)): tb.resetDefaults() tb.close() tb = self.client.sf.createThumbnailStore() assert tb.setPixelsId(long(pix)) after = tb.getThumbnail(i64, i64) assert before != after assert tb.thumbnailExists(i64, i64) assert not tb.isInProgress() elif meth == "set": tb.getThumbnailSet(i64, i64, [long(pix)]) assert get().version.val >= 0
def testPyramidConcurrentAccess(self, tmpdir): """ See ticket:11709 """ all_context = {"omero.group": "-1"} from omero.util import concurrency pix = self.missing_pyramid() rps = self.sf.createRawPixelsStore(all_context) try: # First execution should certainly fail try: rps.setPixelsId(int(pix), True, all_context) assert False, "Should throw!" except omero.MissingPyramidException as mpm: assert int(pix) == mpm.pixelsID # Eventually, however, it should be generated i = 10 success = False while i > 0 and not success: try: rps.setPixelsId(int(pix), True, all_context) success = True except omero.MissingPyramidException as mpm: assert int(pix) == mpm.pixelsID backOff = old_div(mpm.backOff, 1000) event = concurrency.get_event("testRomio") event.wait(backOff) # seconds i -= 1 assert success # Once it's generated, we should be able to concurrencly # access the file without exceptions event = concurrency.get_event("concurrenct_pyramids") sf = self.sf class T(threading.Thread): def run(self): self.success = 0 self.failure = 0 while not event.isSet() and self.success < 10: self.rps = sf.createRawPixelsStore(all_context) try: self.rps.setPixelsId(int(pix), True, all_context) self.success += 1 except Exception: self.failure += 1 raise finally: self.rps.close() threads = [T() for x in range(10)] for t in threads: t.start() event.wait(10) # 10 seconds event.set() for t in threads: t.join() total_successes = sum([t.success for t in threads]) total_failures = sum([t.failure for t in threads]) assert total_successes assert not total_failures finally: rps.close()
success = False while i > 0 and not success: try: rps.setPixelsId(long(pix), True, all_context) success = True except omero.MissingPyramidException, mpm: assert long(pix) == mpm.pixelsID backOff = mpm.backOff / 1000 event = concurrency.get_event("testRomio") event.wait(backOff) # seconds i -= 1 assert success # Once it's generated, we should be able to concurrencly # access the file without exceptions event = concurrency.get_event("concurrenct_pyramids") sf = self.sf class T(threading.Thread): def run(self): self.success = 0 self.failure = 0 while not event.isSet() and self.success < 10: self.rps = sf.createRawPixelsStore(all_context) try: self.rps.setPixelsId(long(pix), True, all_context) self.success += 1 except: self.failure += 1 raise finally:
def __init__(self, client, handle): super(CmdCallback, self).__init__(client, handle) self.t_lock = threading.RLock() self.t_steps = 0 self.t_finished = 0 self.t_event = get_event("CmdCallback")
def keepalive(self, args): import threading from omero.util.concurrency import get_event as get_event class T(threading.Thread): def run(self): while self.client: try: self.client.sf.keepAlive(None) self.event.wait(args.frequency) except exceptions.Exception, e: self.err("Keep alive failed: %s" % str(e)) return t = T() t.client = self.ctx.conn(args) t.event = get_event(name="keepalive") t.start() try: self.ctx.out("Running keep alive every %s seconds" % args.frequency) self.ctx.input("Press enter to cancel.") finally: t.client = None t.event.set() def file(self, args): store = self.store(args) srv, usr, uuid = store.get_current() if srv and usr and uuid: self.ctx.out(str(store.dir / srv / usr / uuid)) def conn(self, properties=None, profile=None, args=None):
def __init__(self, client, handle): self.t_lock = threading.RLock() self.t_steps = 0 self.t_finished = 0 self.t_event = get_event("CmdCallback") super(CmdCallback, self).__init__(client, handle)