def testPyramidConcurrentAccess(self):
"""
See ticket:11709
"""
all_context = {"omero.group": "-1"}
from omero.util import concurrency
pix = self.missing_pyramid(self.root)
rps = self.root.sf.createRawPixelsStore(all_context)
try:
# First execution should certainly fail
try:
rps.setPixelsId(pix.id.val, True, all_context)
assert False, "Should throw!"
except omero.MissingPyramidException, mpm:
assert pix.id.val == mpm.pixelsID
# Eventually, however, it should be generated
i = 10
success = False
while i > 0 and not success:
try:
rps.setPixelsId(pix.id.val, True, all_context)
success = True
except omero.MissingPyramidException, mpm:
assert pix.id.val == mpm.pixelsID
backOff = mpm.backOff/1000
event = concurrency.get_event("testRomio")
event.wait(backOff) # seconds
i -= 1
def testPyramidConcurrentAccess(self, tmpdir):
"""
See ticket:11709
"""
all_context = {"omero.group": "-1"}
from omero.util import concurrency
pix = self.missing_pyramid()
rps = self.sf.createRawPixelsStore(all_context)
try:
# First execution should certainly fail
try:
rps.setPixelsId(long(pix), True, all_context)
assert False, "Should throw!"
except omero.MissingPyramidException, mpm:
assert long(pix) == mpm.pixelsID
# Eventually, however, it should be generated
i = 10
success = False
while i > 0 and not success:
try:
rps.setPixelsId(long(pix), True, all_context)
success = True
except omero.MissingPyramidException, mpm:
assert long(pix) == mpm.pixelsID
backOff = mpm.backOff / 1000
event = concurrency.get_event("testRomio")
event.wait(backOff) # seconds
i -= 1
def testRomioToPyramidWithNegOne(self):
"""
Here we try the above but pass omero.group:-1
to see if we can cause an exception.
"""
all_context = {"omero.group":"-1"}
from omero.util import concurrency
pix = self.missing_pyramid(self.root)
rps = self.root.sf.createRawPixelsStore(all_context)
try:
# First execution should certainly fail
try:
rps.setPixelsId(pix.id.val, True, all_context)
fail("Should throw!")
except omero.MissingPyramidException, mpm:
self.assertEquals(pix.id.val, mpm.pixelsID)
# Eventually, however, it should be generated
i = 10
success = False
while i > 0 and not success:
try:
rps.setPixelsId(pix.id.val, True, all_context)
success = True
except omero.MissingPyramidException, mpm:
self.assertEquals(pix.id.val, mpm.pixelsID)
backOff = mpm.backOff/1000
event = concurrency.get_event("testRomio")
event.wait(backOff) # seconds
i -=1
def testRomioToPyramid(self):
"""
Here we create a pixels that is not big,
then modify its metadata so that it IS big,
in order to trick the service into throwing
us a MissingPyramidException
"""
from omero.util import concurrency
pix = self.missing_pyramid(self.root)
rps = self.root.sf.createRawPixelsStore()
try:
# First execution should certainly fail
try:
rps.setPixelsId(pix.id.val, True)
fail("Should throw!")
except omero.MissingPyramidException, mpm:
self.assertEquals(pix.id.val, mpm.pixelsID)
# Eventually, however, it should be generated
i = 10
success = False
while i > 0 and not success:
try:
rps.setPixelsId(pix.id.val, True)
success = True
except omero.MissingPyramidException, mpm:
self.assertEquals(pix.id.val, mpm.pixelsID)
backOff = mpm.backOff/1000
event = concurrency.get_event("testRomio")
event.wait(backOff) # seconds
i -=1
def testRomioToPyramidWithNegOne(self):
"""
Here we try the above but pass omero.group:-1
to see if we can cause an exception.
"""
all_context = {"omero.group":"-1"}
from omero.util import concurrency
pix = self.missing_pyramid(self.root)
rps = self.root.sf.createRawPixelsStore(all_context)
try:
# First execution should certainly fail
try:
rps.setPixelsId(pix.id.val, True, all_context)
fail("Should throw!")
except omero.MissingPyramidException, mpm:
self.assertEquals(pix.id.val, mpm.pixelsID)
# Eventually, however, it should be generated
i = 10
success = False
while i > 0 and not success:
try:
rps.setPixelsId(pix.id.val, True, all_context)
success = True
except omero.MissingPyramidException, mpm:
self.assertEquals(pix.id.val, mpm.pixelsID)
backOff = mpm.backOff/1000
event = concurrency.get_event("testRomio")
event.wait(backOff) # seconds
i -=1
def keepalive(self, args):
import threading
from omero.util.concurrency import get_event as get_event
class T(threading.Thread):
def run(self):
while self.client:
try:
self.client.sf.keepAlive(None)
self.event.wait(args.frequency)
except Exception as e:
self.err("Keep alive failed: %s" % str(e))
return
t = T()
t.client = self.ctx.conn(args)
t.err = self.ctx.err
t.event = get_event(name="keepalive")
t.start()
try:
self.ctx.out("Running keep alive every %s seconds" %
args.frequency)
self.ctx.input("Press enter to cancel.")
finally:
t.client = None
t.event.set()
def testRomioToPyramid(self, tmpdir):
"""
Here we create a pixels that is not big,
then modify its metadata so that it IS big,
in order to trick the service into throwing
us a MissingPyramidException
"""
from omero.util import concurrency
pix = self.missing_pyramid()
rps = self.sf.createRawPixelsStore()
try:
# First execution should certainly fail
try:
rps.setPixelsId(long(pix), True)
assert False, "Should throw!"
except omero.MissingPyramidException, mpm:
assert long(pix) == mpm.pixelsID
# Eventually, however, it should be generated
i = 10
success = False
while i > 0 and not success:
try:
rps.setPixelsId(long(pix), True)
success = True
except omero.MissingPyramidException, mpm:
assert long(pix) == mpm.pixelsID
backOff = mpm.backOff / 1000
event = concurrency.get_event("testRomio")
event.wait(backOff) # seconds
i -= 1
def __init__(self, controls = None, params = None, prog = sys.argv[0]):
self.controls = controls
if self.controls is None:
self.controls = {}
self.params = params
if self.params is None:
self.params = {}
self.event = get_event(name="CLI")
self.dir = OMERODIR
self.isdebug = DEBUG # This usage will go away and default will be False
self.topics = {"debug":"""
debug options for developers:
The value to the debug argument is a comma-separated list of commands:
* 'debug' prints at the "debug" level. Similar to setting DEBUG=1 in the environment.
* 'trace' runs the command with tracing enabled.
* 'profile' runs the command with profiling enabled.
Only one of "trace" and "profile" can be chosen.
Example:
bin/omero --debug=debug,trace admin start # Debugs at level 1 and prints tracing
bin/omero -d1 admin start # Debugs at level 1
bin/omero -dp admin start # Prints profiling
bin/omero -dt,p admin start # Fails!; can't print tracing and profiling together
bin/omero -d0 admin start # Disables debugging
"""}
self.parser = Parser(prog = prog, description = OMERODOC)
self.subparsers = self.parser_init(self.parser)
def test2RomioToPyramidWithNegOne(self, tmpdir):
"""
Here we try the above but pass omero.group:-1
to see if we can cause an exception.
"""
all_context = {"omero.group": "-1"}
from omero.util import concurrency
pix = self.missing_pyramid()
rps = self.sf.createRawPixelsStore(all_context)
try:
# First execution should certainly fail
try:
rps.setPixelsId(int(pix), True, all_context)
assert False, "Should throw!"
except omero.MissingPyramidException as mpm:
assert int(pix) == mpm.pixelsID
# Eventually, however, it should be generated
i = 10
success = False
while i > 0 and not success:
try:
rps.setPixelsId(int(pix), True, all_context)
success = True
except omero.MissingPyramidException as mpm:
assert int(pix) == mpm.pixelsID
backOff = old_div(mpm.backOff, 1000)
event = concurrency.get_event("testRomio")
event.wait(backOff) # seconds
i -= 1
assert success
finally:
rps.close()
def __init__(self, client, handle, test):
super(TestCB, self).__init__(client, handle)
self.t_test = test
self.t_lock = threading.RLock()
self.t_steps = 0
self.t_finished = 0
self.t_event = get_event("TestCB")
def __init__(self, dir, communicator, getUsedFiles=as_dictionary, ctx=None,
worker_wait=60, worker_count=1, worker_batch=10):
"""
Intialise the instance variables.
"""
self.log = logging.getLogger("fsclient." + __name__)
self.communicator = communicator
self.master = None
#: Reference back to FSServer.
self.serverProxy = None
self.selfProxy = None
self.dropBoxDir = dir
self.host = ""
self.port = 0
self.dirImportWait = 0
self.throttleImport = 5
self.timeToLive = 0
self.timeToIdle = 0
self.readers = ""
self.importArgs = ""
#: Id
self.id = ''
# Overriding methods to allow for simpler testing
self.getUsedFiles = perf(getUsedFiles)
# Threading primitives
self.worker_wait = worker_wait
self.worker_count = worker_count
self.worker_batch = worker_batch
self.event = get_event()
self.queue = Queue.Queue(0)
self.state = MonitorState(self.event)
self.resources = Resources(stop_event=self.event)
if ctx:
# Primarily used for testing
self.ctx = ctx
else:
self.ctx = ServerContext(
server_id="DropBox", communicator=communicator,
stop_event=self.event)
self.resources.add(self.ctx)
self.workers = [
MonitorWorker(
worker_wait, worker_batch, self.event,
self.queue, self.callback)
for x in range(worker_count)]
for worker in self.workers:
worker.start()
self.eventRecord("Directory", self.dropBoxDir)
'propagate': True
},
'': {
'handlers': ['default'],
'level': 'DEBUG',
'propagate': True
}
}
}
# Load custom settings from etc/grid/config.xml
# Tue 2 Nov 2010 11:03:18 GMT -- ticket:3228
from omero.util.concurrency import get_event
CONFIG_XML = os.path.join(OMERO_HOME, 'etc', 'grid', 'config.xml')
count = 10
event = get_event("websettings")
while True:
try:
CUSTOM_SETTINGS = dict()
if os.path.exists(CONFIG_XML):
CONFIG_XML = omero.config.ConfigXml(CONFIG_XML, read_only=True)
CUSTOM_SETTINGS = CONFIG_XML.as_map()
CONFIG_XML.close()
break
except portalocker.LockException:
# logger.error("Exception while loading configuration retrying...",
# exc_info=True)
exctype, value = sys.exc_info()[:2]
count -= 1
if not count:
def keepalive(self, args):
import threading
from omero.util.concurrency import get_event as get_event
class T(threading.Thread):
def run(self):
while self.client:
try:
self.client.sf.keepAlive(None)
self.event.wait(args.frequency)
except Exception, e:
self.err("Keep alive failed: %s" % str(e))
return
t = T()
t.client = self.ctx.conn(args)
t.event = get_event(name="keepalive")
t.start()
try:
self.ctx.out("Running keep alive every %s seconds"
% args.frequency)
self.ctx.input("Press enter to cancel.")
finally:
t.client = None
t.event.set()
def file(self, args):
"""Return the file associated with the current active session"""
store = self.store(args)
srv, usr, uuid, port = store.get_current()
if srv and usr and uuid:
self.ctx.out(str(store.dir / srv / usr / uuid))
'propagate': True
},
'': {
'handlers': ['default'],
'level': 'DEBUG',
'propagate': True
}
}
}
# Load custom settings from etc/grid/config.xml
# Tue 2 Nov 2010 11:03:18 GMT -- ticket:3228
from omero.util.concurrency import get_event
CONFIG_XML = os.path.join(OMERO_HOME, 'etc', 'grid', 'config.xml')
count = 10
event = get_event("websettings")
while True:
try:
CONFIG_XML = omero.config.ConfigXml(CONFIG_XML)
CUSTOM_SETTINGS = CONFIG_XML.as_map()
CONFIG_XML.close()
break
except LockException:
#logger.error("Exception while loading configuration retrying...", exc_info=True)
exctype, value = sys.exc_info()[:2]
count -= 1
if not count:
raise exctype, value
else:
event.wait(1) # Wait a total of 10 seconds
success = False
while i > 0 and not success:
try:
rps.setPixelsId(pix.id.val, True, all_context)
success = True
except omero.MissingPyramidException, mpm:
assert pix.id.val == mpm.pixelsID
backOff = mpm.backOff/1000
event = concurrency.get_event("testRomio")
event.wait(backOff) # seconds
i -= 1
assert success
# Once it's generated, we should be able to concurrencly
# access the file without exceptions
event = concurrency.get_event("concurrenct_pyramids")
root_sf = self.root.sf
class T(threading.Thread):
def run(self):
self.success = 0
self.failure = 0
while not event.isSet() and self.success < 10:
self.rps = root_sf.createRawPixelsStore(all_context)
try:
self.rps.setPixelsId(pix.id.val, True, all_context)
self.success += 1
except:
self.failure += 1
raise
finally:
self.rps.close()
def testThumbnailVersion(self, meth):
assert meth in ("one", "set")
i64 = rint(64)
pix = self.missing_pyramid()
q = ("select tb from Thumbnail tb "
"where tb.pixels.id = %s "
"order by tb.id desc ")
q = q % pix
p = ParametersI().page(0, 1)
get = lambda: self.query.findByQuery(q, p)
# Before anything has been called, there
# should be no thumbnail
assert not get()
# At this stage, there should still be no
# thumbnail
tb = self.client.sf.createThumbnailStore()
if meth == "one":
tb.setPixelsId(long(pix))
tb.resetDefaults()
assert not tb.thumbnailExists(i64, i64)
assert tb.isInProgress()
# As soon as it's requested, it should have a -1
# version to mark pyramid creation as ongoing.
if meth == "one":
before = tb.getThumbnail(i64, i64)
assert not tb.thumbnailExists(i64, i64)
assert tb.isInProgress()
elif meth == "set":
before = tb.getThumbnailSet(i64, i64, [long(pix)])
before = before[long(pix)]
assert get().version.val == -1
# Now we wait until the pyramid has been created
# and test that a proper version has been set.
event = get_event("test_thumbs")
secs = 20
rps = self.client.sf.createRawPixelsStore()
for x in range(secs):
try:
rps.setPixelsId(long(pix), True)
event = None
break
except MissingPyramidException:
event.wait(1)
if event:
assert "Pyramid was not generated %ss" % secs
if meth == "one":
# Re-load the thumbnail store now that
# the pyramid is generated.
tb.close()
tb = self.client.sf.createThumbnailStore()
if not tb.setPixelsId(long(pix)):
tb.resetDefaults()
tb.close()
tb = self.client.sf.createThumbnailStore()
assert tb.setPixelsId(long(pix))
after = tb.getThumbnail(i64, i64)
assert before != after
assert tb.thumbnailExists(i64, i64)
assert not tb.isInProgress()
elif meth == "set":
tb.getThumbnailSet(i64, i64, [long(pix)])
assert get().version.val >= 0
def testPyramidConcurrentAccess(self, tmpdir):
"""
See ticket:11709
"""
all_context = {"omero.group": "-1"}
from omero.util import concurrency
pix = self.missing_pyramid()
rps = self.sf.createRawPixelsStore(all_context)
try:
# First execution should certainly fail
try:
rps.setPixelsId(int(pix), True, all_context)
assert False, "Should throw!"
except omero.MissingPyramidException as mpm:
assert int(pix) == mpm.pixelsID
# Eventually, however, it should be generated
i = 10
success = False
while i > 0 and not success:
try:
rps.setPixelsId(int(pix), True, all_context)
success = True
except omero.MissingPyramidException as mpm:
assert int(pix) == mpm.pixelsID
backOff = old_div(mpm.backOff, 1000)
event = concurrency.get_event("testRomio")
event.wait(backOff) # seconds
i -= 1
assert success
# Once it's generated, we should be able to concurrencly
# access the file without exceptions
event = concurrency.get_event("concurrenct_pyramids")
sf = self.sf
class T(threading.Thread):
def run(self):
self.success = 0
self.failure = 0
while not event.isSet() and self.success < 10:
self.rps = sf.createRawPixelsStore(all_context)
try:
self.rps.setPixelsId(int(pix), True, all_context)
self.success += 1
except Exception:
self.failure += 1
raise
finally:
self.rps.close()
threads = [T() for x in range(10)]
for t in threads:
t.start()
event.wait(10) # 10 seconds
event.set()
for t in threads:
t.join()
total_successes = sum([t.success for t in threads])
total_failures = sum([t.failure for t in threads])
assert total_successes
assert not total_failures
finally:
rps.close()
success = False
while i > 0 and not success:
try:
rps.setPixelsId(long(pix), True, all_context)
success = True
except omero.MissingPyramidException, mpm:
assert long(pix) == mpm.pixelsID
backOff = mpm.backOff / 1000
event = concurrency.get_event("testRomio")
event.wait(backOff) # seconds
i -= 1
assert success
# Once it's generated, we should be able to concurrencly
# access the file without exceptions
event = concurrency.get_event("concurrenct_pyramids")
sf = self.sf
class T(threading.Thread):
def run(self):
self.success = 0
self.failure = 0
while not event.isSet() and self.success < 10:
self.rps = sf.createRawPixelsStore(all_context)
try:
self.rps.setPixelsId(long(pix), True, all_context)
self.success += 1
except:
self.failure += 1
raise
finally:
def testThumbnailVersion(self, meth):
assert meth in ("one", "set")
i64 = rint(64)
pix = self.missing_pyramid()
q = ("select tb from Thumbnail tb "
"where tb.pixels.id = %s "
"order by tb.id desc ")
q = q % pix
p = ParametersI().page(0, 1)
get = lambda: self.query.findByQuery(q, p)
# Before anything has been called, there
# should be no thumbnail
assert not get()
# At this stage, there should still be no
# thumbnail
tb = self.client.sf.createThumbnailStore()
if meth == "one":
tb.setPixelsId(long(pix))
tb.resetDefaults()
assert not tb.thumbnailExists(i64, i64)
assert tb.isInProgress()
# As soon as it's requested, it should have a -1
# version to mark pyramid creation as ongoing.
if meth == "one":
before = tb.getThumbnail(i64, i64)
assert not tb.thumbnailExists(i64, i64)
assert tb.isInProgress()
elif meth == "set":
before = tb.getThumbnailSet(i64, i64, [long(pix)])
before = before[long(pix)]
assert get().version.val == -1
# Now we wait until the pyramid has been created
# and test that a proper version has been set.
event = get_event("test_thumbs")
secs = 20
rps = self.client.sf.createRawPixelsStore()
for x in range(secs):
try:
rps.setPixelsId(long(pix), True)
event = None
break
except MissingPyramidException:
event.wait(1)
if event:
assert "Pyramid was not generated %ss" % secs
if meth == "one":
# Re-load the thumbnail store now that
# the pyramid is generated.
tb.close()
tb = self.client.sf.createThumbnailStore()
if not tb.setPixelsId(long(pix)):
tb.resetDefaults()
tb.close()
tb = self.client.sf.createThumbnailStore()
assert tb.setPixelsId(long(pix))
after = tb.getThumbnail(i64, i64)
assert before != after
assert tb.thumbnailExists(i64, i64)
assert not tb.isInProgress()
elif meth == "set":
tb.getThumbnailSet(i64, i64, [long(pix)])
assert get().version.val >= 0
def __init__(self, client, handle):
super(CmdCallback, self).__init__(client, handle)
self.t_lock = threading.RLock()
self.t_steps = 0
self.t_finished = 0
self.t_event = get_event("CmdCallback")
def keepalive(self, args):
import threading
from omero.util.concurrency import get_event as get_event
class T(threading.Thread):
def run(self):
while self.client:
try:
self.client.sf.keepAlive(None)
self.event.wait(args.frequency)
except exceptions.Exception, e:
self.err("Keep alive failed: %s" % str(e))
return
t = T()
t.client = self.ctx.conn(args)
t.event = get_event(name="keepalive")
t.start()
try:
self.ctx.out("Running keep alive every %s seconds" % args.frequency)
self.ctx.input("Press enter to cancel.")
finally:
t.client = None
t.event.set()
def file(self, args):
store = self.store(args)
srv, usr, uuid = store.get_current()
if srv and usr and uuid:
self.ctx.out(str(store.dir / srv / usr / uuid))
def conn(self, properties=None, profile=None, args=None):
def __init__(self, client, handle):
self.t_lock = threading.RLock()
self.t_steps = 0
self.t_finished = 0
self.t_event = get_event("CmdCallback")
super(CmdCallback, self).__init__(client, handle)
