def initialize(self, cols, metadata = None):
"""
"""
if metadata is None: metadata = {}
if self.__initialized:
raise omero.ValidationException(None, None, "Already initialized.")
if not cols:
raise omero.ApiUsageException(None, None, "No columns provided")
for c in cols:
if not c.name:
raise omero.ApiUsageException(None, None, "Column unnamed: %s" % c)
self.__definition = columns2definition(cols)
self.__ome = self.__hdf_file.createGroup("/", "OME")
self.__mea = self.__hdf_file.createTable(self.__ome, "Measurements", self.__definition)
self.__types = [ x.ice_staticId() for x in cols ]
self.__descriptions = [ (x.description != None) and x.description or "" for x in cols ]
self.__hdf_file.createArray(self.__ome, "ColumnTypes", self.__types)
self.__hdf_file.createArray(self.__ome, "ColumnDescriptions", self.__descriptions)
self.__mea.attrs.version = "v1"
self.__mea.attrs.initialized = time.time()
if metadata:
for k, v in metadata.items():
self.__mea.attrs[k] = v
# See attrs._f_list("user") to retrieve these.
self.__mea.flush()
self.__hdf_file.flush()
self.__initialized = True
def add_meta_map(self, m, replace=False, init=False):
if not init:
if int(self.__getversion()) < 2:
# Metadata methods were generally broken for v1 tables so
# the introduction of internal metadata attributes is unlikely
# to affect anyone.
# https://trac.openmicroscopy.org/ome/ticket/12606
msg = 'Tables metadata is only supported for OMERO.tables >= 2'
self.logger.error(msg)
raise omero.ApiUsageException(None, None, msg)
self.__initcheck()
for k, v in m.iteritems():
if internal_attr(k):
raise omero.ApiUsageException(
None, None, "Reserved attribute name: %s" % k)
if not isinstance(v, (
omero.RString, omero.RLong, omero.RInt, omero.RFloat)):
raise omero.ValidationException(
"Unsupported type: %s" % type(v))
attr = self.__mea.attrs
if replace:
for f in list(attr._v_attrnamesuser):
if init or not internal_attr(f):
del attr[f]
if not m:
return
for k, v in m.iteritems():
# This uses the default pytables type conversion, which may
# convert it to a numpy type or keep it as a native Python type
attr[k] = unwrap(v)
def fsEventHappened(self, monitorid, eventList, current=None):
"""
Primary monitor client callback.
If new files appear on the watch, the list is sent as an argument.
The id should match for the events to be relevant.
At the moment each file type is treated as a special case. The
number of special cases is likely to explode and so a different
approach is needed. That will be easier with more knowledge of the
different multi-file formats.
:Parameters:
id : string
A string uniquely identifying the OMERO.fs Watch created
by the OMERO.fs Server.
eventList : list<string>
A list of events, in the current implementation this is
a list of strings representing the full path names of new
files.
current
An ICE context, this parameter is required to be present
in an ICE callback.
:return: No explicit return value
"""
# ! Set import to dummy mode for testing purposes.
# self.importFile = self.dummyImportFile
# ! If the above line is not commented out nothing will import.
if self.id != monitorid:
self.warnAndThrow(
omero.ApiUsageException(),
"Unknown fs server id: %s", monitorid)
self.eventRecord("Batch", len(eventList))
for fileInfo in eventList:
fileId = fileInfo.fileId
if not fileId:
self.warnAndThrow(omero.ApiUsageException(), "Empty fieldId")
self.eventRecord(fileInfo.type, fileId)
# Checking name first since it's faster
exName = self.getExperimenterFromPath(fileId)
if exName and self.userExists(exName):
# Creation or modification handled by state/timeout system
if (str(fileInfo.type) == "Create"
or str(fileInfo.type) == "Modify"):
self.queue.put(fileInfo)
else:
self.log.info(
"Event not Create or Modify, presently ignored.")
def unregisterCallback(self, callback, current=None):
try:
id = callback.ice_getIdentity()
key = "%s/%s" % (id.category, id.name)
if key not in self.callback:
raise omero.ApiUsageException(
None, None, "No callback registered with id: %s" % key)
del self.callbacks[key]
self.logger.debug("Removed callback: %s", key)
except Exception, e:
msg = "Failed to remove callback: %s. Reason: %s" % (callback, e)
self.logger.debug(msg)
raise omero.ApiUsageException(None, None, msg)
def getSession(self, recreate = True):
"""
Returns the ServiceFactoryPrx configured for the context if
available. If the context was not configured for sessions,
an ApiUsageException will be thrown: servants should know
whether or not they were configured for sessions.
See Servant(..., needs_session = True)
Otherwise, if there is no ServiceFactoryPrx, an attempt will
be made to create one if recreate == True. If the value is None
or non can be recreated, an InternalException will be thrown.
TODO : currently no arguments are provided for re-creating these,
but also not in Servant.__init__
"""
if not self.hasSession():
raise omero.ApiUsageException("Not configured for server connection")
if self.session:
try:
self.session.keepAlive(None)
except Ice.CommunicatorDestroyedException:
self.session = None # Ignore
except exceptions.Exception, e:
self.logger.warn("Connection failure: %s" % e)
self.session = None
def addOrThrow(self, hdfpath, hdfstorage):
if hdfpath in self.__paths:
raise omero.LockTimeout(
None, None, "Path already in HdfList: %s" % hdfpath)
parent = path(hdfpath).parent
if not parent.exists():
raise omero.ApiUsageException(
None, None, "Parent directory does not exist: %s" % parent)
hdffile = hdfstorage.openfile("a")
fileno = hdffile.fileno()
try:
portalocker.lockno(
fileno, portalocker.LOCK_NB | portalocker.LOCK_EX)
except portalocker.LockException:
hdffile.close()
raise omero.LockTimeout(
None, None,
"Cannot acquire exclusive lock on: %s" % hdfpath, 0)
except:
hdffile.close()
raise
if fileno in self.__filenos.keys():
hdffile.close()
raise omero.LockTimeout(
None, None, "File already opened by process: %s" % hdfpath, 0)
else:
self.__filenos[fileno] = hdfstorage
self.__paths[hdfpath] = hdfstorage
return hdffile
def descriptor(self, pos):
# During initialization, size might be zero
if pos is None:
return tables.Int64Col(pos=pos)
if self.size < 1:
raise omero.ApiUsageException(
None, None, "Array length must be > 0 (Column: %s)" % self.name)
return tables.Int64Col(pos=pos, shape=self.size)
def incr(self, table):
sz = len(self.__tables)
self.logger.info("Size: %s - Attaching %s to %s" % (sz, table, self.__hdf_path))
if table in self.__tables:
self.logger.warn("Already added")
raise omero.ApiUsageException(None, None, "Already added")
self.__tables.append(table)
return sz + 1
def descriptor(self, pos):
# During initialization, size might be zero
# to prevent exceptions we temporarily assume size 1
if pos is None:
return tables.StringCol(pos=pos, itemsize=1)
if self.size < 1:
raise omero.ApiUsageException(
None, None, "String size must be > 0 (Column: %s)" % self.name)
return tables.StringCol(pos=pos, itemsize=self.size)
def readCoordinates(self, rowNumbers, current = None):
self.logger.info("%s.readCoordinates(size=%s)", self, slen(rowNumbers))
try:
return self.storage.readCoordinates(self.stamp, rowNumbers, current)
except tables.HDF5ExtError, err:
aue = omero.ApiUsageException()
aue.message = "Error reading coordinates. Most likely out of range"
aue.serverStackTrace = "".join(traceback.format_exc())
aue.serverExceptionClass = str(err.__class__.__name__)
raise aue
def getWhereList(self, stamp, condition, variables, unused, start, stop, step):
self.__initcheck()
try:
return self.__mea.getWhereList(condition, variables, None, start, stop, step).tolist()
except (exceptions.NameError, exceptions.SyntaxError, exceptions.TypeError, exceptions.ValueError), err:
aue = omero.ApiUsageException()
aue.message = "Bad condition: %s, %s" % (condition, variables)
aue.serverStackTrace = "".join(traceback.format_exc())
aue.serverExceptionClass = str(err.__class__.__name__)
raise aue
def decr(self, table):
sz = len(self.__tables)
self.logger.info("Size: %s - Detaching %s from %s", sz, table, self.__hdf_path)
if not (table in self.__tables):
self.logger.warn("Unknown table")
raise omero.ApiUsageException(None, None, "Unknown table")
self.__tables.remove(table)
if sz <= 1:
self.cleanup()
return sz - 1
def __sizecheck(self, colNumbers, rowNumbers):
if colNumbers is not None:
if len(colNumbers) > 0:
maxcol = max(colNumbers)
totcol = self.__width()
if maxcol >= totcol:
raise omero.ApiUsageException(None, None, "Column overflow: %s >= %s" % (maxcol, totcol))
else:
raise omero.ApiUsageException(None, None, "Columns not specified: %s" % colNumbers)
if rowNumbers is not None:
if len(rowNumbers) > 0:
maxrow = max(rowNumbers)
totrow = self.__length()
if maxrow >= totrow:
raise omero.ApiUsageException(None, None, "Row overflow: %s >= %s" % (maxrow, totrow))
else:
raise omero.ApiUsageException(None, None, "Rows not specified: %s" % rowNumbers)
def initialize(self, cols, metadata=None):
"""
"""
if metadata is None:
metadata = {}
if self.__initialized:
raise omero.ValidationException(None, None, "Already initialized.")
if not cols:
raise omero.ApiUsageException(None, None, "No columns provided")
for c in cols:
if not c.name:
raise omero.ApiUsageException(
None, None, "Column unnamed: %s" % c)
if internal_attr(c.name):
raise omero.ApiUsageException(
None, None, "Reserved column name: %s" % c.name)
self.__definition = columns2definition(cols)
self.__ome = self.__hdf_file.createGroup("/", "OME")
self.__mea = self.__hdf_file.createTable(
self.__ome, "Measurements", self.__definition)
self.__types = [x.ice_staticId() for x in cols]
self.__descriptions = [
(x.description is not None) and x.description or "" for x in cols]
self.__hdf_file.createArray(self.__ome, "ColumnTypes", self.__types)
self.__hdf_file.createArray(
self.__ome, "ColumnDescriptions", self.__descriptions)
md = {}
if metadata:
md = metadata.copy()
md['__version'] = VERSION
md['__initialized'] = time.time()
self.add_meta_map(md, replace=True, init=True)
self.__hdf_file.flush()
self.__initialized = True
def read(self, colNumbers, start, stop, current=None):
self.logger.info("%s.read(%s, %s, %s)", self, colNumbers, start, stop)
if start == 0 and stop == 0:
stop = None
try:
return self.storage.read(self.stamp, colNumbers, start, stop,
current)
except tables.HDF5ExtError as err:
aue = omero.ApiUsageException()
aue.message = "Error reading coordinates. Most likely out of range"
aue.serverStackTrace = "".join(traceback.format_exc())
aue.serverExceptionClass = str(err.__class__.__name__)
raise aue
def columns2definition(cols):
"""
Takes a list of columns and converts them into a map
from names to tables.* column descriptors
"""
definition = {}
for i in range(len(cols)):
column = cols[i]
instance = column.descriptor(pos=i)
if column.name in definition:
raise omero.ApiUsageException(
None, None, "Duplicate column name: %s" % column.name)
definition[column.name] = instance
# Descriptions are handled separately
return definition
def getWhereList(self, stamp, condition, variables, unused, start, stop,
step):
self.__initcheck()
try:
condvars = variables
if variables:
for key, value in condvars.items():
if isinstance(value, str):
condvars[key] = getattr(self.__mea.cols, value)
return self.__mea.get_where_list(condition, condvars, None, start,
stop, step).tolist()
except (NameError, SyntaxError, TypeError, ValueError) as err:
aue = omero.ApiUsageException()
aue.message = "Bad condition: %s, %s" % (condition, variables)
aue.serverStackTrace = "".join(traceback.format_exc())
aue.serverExceptionClass = str(err.__class__.__name__)
raise aue
def addOrThrow(self, hdfpath, hdfstorage):
if hdfpath in self.__locks:
raise omero.LockTimeout(None, None, "Path already in HdfList: %s" % hdfpath)
parent = path(hdfpath).parent
if not parent.exists():
raise omero.ApiUsageException(None, None, "Parent directory does not exist: %s" % parent)
lock = None
try:
lock = open(hdfpath, "a+")
portalocker.lock(lock, portalocker.LOCK_NB|portalocker.LOCK_EX)
self.__locks[hdfpath] = lock
except portalocker.LockException, le:
if lock:
lock.close()
raise omero.LockTimeout(None, None, "Cannot acquire exclusive lock on: %s" % hdfpath, 0)
def registerCallback(self, callback, current=None):
try:
id = callback.ice_getIdentity()
key = "%s/%s" % (id.category, id.name)
callback = callback.ice_oneway()
callback = self.callback_cast(callback)
if not callback:
e = "Callback is invalid"
else:
self.callbacks[key] = callback
self.logger.debug("Added callback: %s", key)
return
except Exception as ex:
e = ex
# Only reached on failure
msg = "Failed to add callback: %s. Reason: %s" % (callback, e)
self.logger.debug(msg)
raise omero.ApiUsageException(None, None, msg)
def lookup(self, job):
sf = self.internal_session()
gid = job.details.group.id.val
handle = WithGroup(sf.createJobHandle(), gid)
try:
handle.attach(job.id.val)
if handle.jobFinished():
handle.close()
raise omero.ApiUsageException("Job already finished.")
prx = WithGroup(sf.getScriptService(), gid)
file = prx.validateScript(job, self.accepts_list)
except omero.SecurityViolation, sv:
self.logger.debug(
"SecurityViolation on validate job %s from group %s",
job.id.val, gid)
file = None
def deactivate(self):
"""
Cleans up the temporary directory used by the process, and terminates
the Popen process if running.
"""
if not self.isActive():
raise omero.ApiUsageException(None, None, "Not active")
if self.stopped:
# Prevent recursion since we are reusing kill & cancel
return
self.stopped = time.time()
d_start = time.time()
self.status("Deactivating")
# None of these should throw, but just in case
try:
self.shutdown() # Calls cancel & kill which recall this method!
self.popen = None # Now we are finished
client = self.tmp_client()
try:
self.set_job_status(client)
self.cleanup_output()
self.upload_output(client) # Important!
self.cleanup_tmpdir()
finally:
if client:
client.__del__() # Safe closeSession
except Exception:
self.logger.error("FAILED TO CLEANUP pid=%s (%s)",
self.pid,
self.uuid,
exc_info=True)
d_stop = time.time()
elapsed = int(self.stopped - self.started)
d_elapsed = int(d_stop - d_start)
self.status("Lived %ss. Deactivation took %ss." % (elapsed, d_elapsed))
def activate(self):
"""
Process creation has to wait until all external downloads, etc
are finished.
"""
if self.isActive():
raise omero.ApiUsageException(None, None, "Already activated")
self.stdout = open(str(self.stdout_path), "w")
self.stderr = open(str(self.stderr_path), "w")
self.popen = self.Popen(self.command(),
cwd=str(self.dir),
env=self.env(),
stdout=self.stdout,
stderr=self.stderr)
self.pid = self.popen.pid
self.started = time.time()
self.stopped = None
self.status("Activated")
def __initcheck(self):
if not self.__initialized:
raise omero.ApiUsageException(None, None, "Not yet initialized")
def addColumn(self, col, current=None):
self.assert_write()
raise omero.ApiUsageException(None, None, "NYI")
aue.message = "Error reading coordinates. Most likely out of range"
aue.serverStackTrace = "".join(traceback.format_exc())
aue.serverExceptionClass = str(err.__class__.__name__)
raise aue
@remoted
@perf
def read(self, colNumbers, start, stop, current=None):
self.logger.info("%s.read(%s, %s, %s)", self, colNumbers, start, stop)
if start == 0L and stop == 0L:
stop = None
try:
return self.storage.read(self.stamp, colNumbers,
start, stop, current)
except tables.HDF5ExtError, err:
aue = omero.ApiUsageException()
aue.message = "Error reading coordinates. Most likely out of range"
aue.serverStackTrace = "".join(traceback.format_exc())
aue.serverExceptionClass = str(err.__class__.__name__)
raise aue
@remoted
@perf
def slice(self, colNumbers, rowNumbers, current=None):
self.logger.info(
"%s.slice(size=%s, size=%s)", self,
slen(colNumbers), slen(rowNumbers))
return self.storage.slice(self.stamp, colNumbers, rowNumbers, current)
# TABLES WRITE API ===========================
id = callback.ice_getIdentity()
key = "%s/%s" % (id.category, id.name)
callback = callback.ice_oneway()
callback = self.callback_cast(callback)
if not callback:
e = "Callback is invalid"
else:
self.callbacks[key] = callback
self.logger.debug("Added callback: %s", key)
return
except Exception, ex:
e = ex
# Only reached on failure
msg = "Failed to add callback: %s. Reason: %s" % (callback, e)
self.logger.debug(msg)
raise omero.ApiUsageException(None, None, msg)
@remoted
@locked
def unregisterCallback(self, callback, current=None):
try:
id = callback.ice_getIdentity()
key = "%s/%s" % (id.category, id.name)
if key not in self.callback:
raise omero.ApiUsageException(
None, None, "No callback registered with id: %s" % key)
del self.callbacks[key]
self.logger.debug("Removed callback: %s", key)
except Exception, e:
msg = "Failed to remove callback: %s. Reason: %s" % (callback, e)
self.logger.debug(msg)
def testTablesIGetDirGetsRepoGetsSFCantFindRepoObject(self):
self.repofile(self.sf.db_uuid)
self.sf.return_values.append(
omero.ApiUsageException(None, None, "Can't Find"))
pytest.raises(omero.ApiUsageException, omero.tables.TablesI, self.ctx)
def process(self,
client,
session,
job,
current,
params,
properties=None,
iskill=True):
"""
session: session uuid, used primarily if client is None
client: an omero.client object which should be attached to a session
"""
if properties is None:
properties = {}
if not session or not job or not job.id:
raise omero.ApiUsageException("No null arguments")
file, handle = self.lookup(job)
try:
if not file:
raise omero.ApiUsageException(
None, None,
"Job should have one executable file attached.")
sf = self.internal_session()
if params:
self.logger.debug("Checking params for job %s" % job.id.val)
svc = sf.getSessionService()
inputs = svc.getInputs(session)
errors = omero.scripts.validate_inputs(params, inputs, svc,
session)
if errors:
errors = "Invalid parameters:\n%s" % errors
raise omero.ValidationException(None, None, errors)
properties["omero.job"] = str(job.id.val)
properties["omero.user"] = session
properties["omero.pass"] = session
properties["Ice.Default.Router"] = \
client.getProperty("Ice.Default.Router")
launcher, ProcessClass = self.find_launcher(current)
process = ProcessClass(self.ctx,
launcher,
properties,
params,
iskill,
omero_home=self.omero_home)
self.resources.add(process)
# client.download(file, str(process.script_path))
scriptText = sf.getScriptService().getScriptText(file.id.val)
process.script_path.write_bytes(scriptText)
self.logger.info("Downloaded file: %s" % file.id.val)
s = client.sha1(str(process.script_path))
if not s == file.hash.val:
msg = "Sha1s don't match! expected %s, found %s" \
% (file.hash.val, s)
self.logger.error(msg)
process.cleanup()
raise omero.InternalException(None, None, msg)
else:
process.activate()
handle.setStatus("Running")
id = None
if self.category:
id = Ice.Identity()
id.name = "Process-%s" % uuid.uuid4()
id.category = self.category
prx = self.ctx.add_servant(current, process, ice_identity=id)
return omero.grid.ProcessPrx.uncheckedCast(prx), process
finally:
handle.close()
