def __init__(self, dir, communicator, getUsedFiles=as_dictionary, ctx=None,
worker_wait=60, worker_count=1, worker_batch=10):
"""
Intialise the instance variables.
"""
self.log = logging.getLogger("fsclient." + __name__)
self.communicator = communicator
self.master = None
#: Reference back to FSServer.
self.serverProxy = None
self.selfProxy = None
self.dropBoxDir = dir
self.host = ""
self.port = 0
self.dirImportWait = 0
self.throttleImport = 5
self.timeToLive = 0
self.timeToIdle = 0
self.readers = ""
self.importArgs = ""
#: Id
self.id = ''
# Overriding methods to allow for simpler testing
self.getUsedFiles = perf(getUsedFiles)
# Threading primitives
self.worker_wait = worker_wait
self.worker_count = worker_count
self.worker_batch = worker_batch
self.event = get_event()
self.queue = Queue.Queue(0)
self.state = MonitorState(self.event)
self.resources = Resources(stop_event=self.event)
if ctx:
# Primarily used for testing
self.ctx = ctx
else:
self.ctx = ServerContext(
server_id="DropBox", communicator=communicator,
stop_event=self.event)
self.resources.add(self.ctx)
self.workers = [
MonitorWorker(
worker_wait, worker_batch, self.event,
self.queue, self.callback)
for x in range(worker_count)]
for worker in self.workers:
worker.start()
self.eventRecord("Directory", self.dropBoxDir)
class MonitorClientI(monitors.MonitorClient):
"""
Implementation of the MonitorClient.
The interface of the MonitorClient is defined in omerofs.ice and
contains the single callback below.
"""
def __init__(self, dir, communicator, getUsedFiles = as_dictionary, ctx = None,\
worker_wait = 60, worker_count = 1, worker_batch = 10):
"""
Intialise the instance variables.
"""
self.log = logging.getLogger("fsclient."+__name__)
self.communicator = communicator
self.master = None
#: Reference back to FSServer.
self.serverProxy = None
self.selfProxy = None
self.dropBoxDir = dir
self.host = ""
self.port = 0
self.dirImportWait = 0
self.throttleImport = 5
self.timeToLive = 0
self.timeToIdle = 0
self.readers = ""
self.importArgs = ""
#: Id
self.id = ''
# Overriding methods to allow for simpler testing
self.getUsedFiles = perf(getUsedFiles)
# Threading primitives
self.worker_wait = worker_wait
self.worker_count = worker_count
self.worker_batch = worker_batch
self.event = get_event()
self.queue = Queue.Queue(0)
self.state = MonitorState(self.event)
self.resources = Resources(stop_event = self.event)
if ctx:
# Primarily used for testing
self.ctx = ctx
else:
self.ctx = ServerContext(server_id = "DropBox", communicator = communicator, stop_event = self.event)
self.resources.add(self.ctx)
self.workers = [MonitorWorker(worker_wait, worker_batch, self.event, self.queue, self.callback) for x in range(worker_count)]
for worker in self.workers:
worker.start()
self.eventRecord("Directory", self.dropBoxDir)
@perf
def stop(self):
"""
Shutdown this servant
"""
self.event.set() # Marks everything as stopping
# Shutdown all workers first, otherwise
# there will be contention on the state
workers = self.workers
self.workers = None
if workers:
self.log.info("Joining workers...")
for x in workers:
x.join()
try:
state = self.state
self.state = None
self.log.info("Stopping state...")
if state: state.stop()
except:
self.log.exception("Error stopping state")
try:
resources = self.resources
self.resources = None
self.log.info("Cleaning up resources state...")
if resources: resources.cleanup()
except:
self.log.exception("Error cleaning resources")
def __del__(self):
self.stop()
#
# Called by server threads.
#
@remoted
@perf
def fsEventHappened(self, monitorid, eventList, current=None):
"""
Primary monitor client callback.
If new files appear on the watch, the list is sent as an argument.
The id should match for the events to be relevant.
At the moment each file type is treated as a special case. The number of
special cases is likely to explode and so a different approach is needed.
That will be easier with more knowledge of the different multi-file formats.
:Parameters:
id : string
A string uniquely identifying the OMERO.fs Watch created
by the OMERO.fs Server.
eventList : list<string>
A list of events, in the current implementation this is
a list of strings representing the full path names of new files.
current
An ICE context, this parameter is required to be present
in an ICE callback.
:return: No explicit return value
"""
# ############## ! Set import to dummy mode for testing purposes.
# self.importFile = self.dummyImportFile
# ############## ! If the above line is not commented out nothing will import.
if self.id != monitorid:
self.warnAndThrow(omero.ApiUsageException(), "Unknown fs server id: %s", monitorid)
self.eventRecord("Batch", len(eventList))
for fileInfo in eventList:
fileId = fileInfo.fileId
if not fileId:
self.warnAndThrow(omero.ApiUsageException(), "Empty fieldId")
self.eventRecord(fileInfo.type, fileId)
# Checking name first since it's faster
exName = self.getExperimenterFromPath(fileId)
if exName and self.userExists(exName):
# Creation or modification handled by state/timeout system
if str(fileInfo.type) == "Create" or str(fileInfo.type) == "Modify":
self.queue.put(fileInfo)
else:
self.log.info("Event not Create or Modify, presently ignored.")
#
# Called by worker threads.
#
@perf
def callback(self, ids):
try:
self.log.info("Getting filesets on : %s", ids)
fileSets = self.getUsedFiles(list(ids), readers=self.readers)
self.eventRecord("Filesets", str(fileSets))
except:
self.log.exception("Failed to get filesets")
fileSets = None
if fileSets:
self.state.update(fileSets, self.dirImportWait, self.importFileWrapper)
#
# Called from state callback (timer)
#
def importFileWrapper(self, fileId):
"""
Wrapper method which allows plugging error handling code around
the main call to importFile. In all cases, the key will be removed
on execution.
"""
self.state.clear(fileId)
exName = self.getExperimenterFromPath(fileId)
self.importFile(fileId, exName)
#
# Helpers
#
def getExperimenterFromPath(self, fileId=""):
"""
Extract experimenter name from path. If the experimenter
cannot be extracted, then null will be returned, in which
case no import should take place.
"""
fileId = pathModule.path(fileId)
exName = None
parpath = fileId.parpath(self.dropBoxDir)
if parpath and len(parpath) >= 2:
fileParts = fileId.splitall()
i = -1 * len(parpath)
fileParts = fileParts[i:]
# For .../DropBox/user structure
if len(fileParts) >= 2:
exName = fileParts[0]
# For .../DropBox/u/user structure
#if len(fileParts) >= 3:
# exName = fileParts[1]
if not exName:
self.log.error("File added outside user directories: %s" % fileId)
return exName
def loginUser(self, exName):
"""
Logins in the given user and returns the client
"""
if not self.ctx.hasSession():
self.ctx.newSession()
sf = None
try:
sf = self.ctx.getSession()
except:
self.log.exception("Failed to get sf \n")
if not sf:
self.log.error("No connection")
return None
p = omero.sys.Principal()
p.name = exName
p.group = "user"
p.eventType = "User"
try:
exp = sf.getAdminService().lookupExperimenter(exName)
sess = sf.getSessionService().createSessionWithTimeouts(p, self.timeToLive, self.timeToIdle)
return sess.uuid.val
except omero.ApiUsageException:
self.log.info("User unknown: %s", exName)
return None
except:
self.log.exception("Unknown exception during loginUser")
return None
def userExists(self, exName):
"""
Tests if the given user exists.
"""
if not self.ctx.hasSession():
self.ctx.newSession()
sf = None
try:
sf = self.ctx.getSession()
except:
self.log.exception("Failed to get sf \n")
if not sf:
self.log.error("No connection")
return False
try:
exp = sf.getAdminService().lookupExperimenter(exName)
return True
except omero.ApiUsageException:
self.log.info("User unknown: %s", exName)
return False
except:
self.log.exception("Unknown exception during loginUser")
return False
@perf
def importFile(self, fileName, exName):
"""
Import file or directory using 'bin/omero importer'
This method is solely responsible for logging the user in,
attempting (possibly multiply) an import, logging and
throwing an exception if necessary.
"""
key = self.loginUser(exName)
if not key:
self.log.info("File not imported: %s", fileName)
return
try:
self.state.appropriateWait(self.throttleImport) # See ticket:5739
self.log.info("Importing %s (session=%s)", fileName, key)
imageId = []
t = create_path("dropbox", "err")
to = create_path("dropbox", "out")
cli = omero.cli.CLI()
cli.loadplugins()
cmd = ["-s", self.host, "-p", str(self.port), "-k", key, "import"]
cmd.extend([str("---errs=%s"%t), str("---file=%s"%to), "--", "--agent=dropbox"])
cmd.extend(shlex.split(self.importArgs))
cmd.append(fileName)
logging.debug("cli.invoke(%s)" % cmd)
cli.invoke(cmd)
retCode = cli.rv
if retCode == 0:
self.log.info("Import of %s completed (session=%s)", fileName, key)
if to.exists():
f = open(str(to),"r")
lines = f.readlines()
f.close()
if len(lines) > 0:
for line in lines:
imageId.append(line.strip())
else:
self.log.error("No lines in output file. No image ID.")
else:
self.log.error("%s not found !" % to)
else:
self.log.error("Import of %s failed=%s (session=%s)", fileName, str(retCode), key)
self.log.error("***** start of output from importer-cli to stderr *****")
if t.exists():
f = open(str(t),"r")
lines = f.readlines()
f.close()
for line in lines:
self.log.error(line.strip())
else:
self.log.error("%s not found !" % t)
self.log.error("***** end of output from importer-cli *****")
finally:
remove_path(t)
remove_path(to)
return imageId
#
# Setters
#
def dummyImportFile(self, fileName, exName):
"""
Log a potential import for test purposes
"""
self.log.info("***DUMMY IMPORT*** Would have tried to import: %s ", fileName)
def setMaster(self, master):
"""
Setter for FSDropBox
:Parameters:
master : DropBox
DropBox Server
:return: No explicit return value.
"""
self.master = master
def setServerProxy(self, serverProxy):
"""
Setter for serverProxy
:Parameters:
serverProxy : monitors.MonitorServerPrx
proxy to remote server object
:return: No explicit return value.
"""
self.serverProxy = serverProxy
def setSelfProxy(self, selfProxy):
"""
Setter for serverProxy
:Parameters:
selfProxy : monitors.MonitorClientPrx
proxy to this client object
:return: No explicit return value.
"""
self.selfProxy = selfProxy
def setId(self, id):
"""
Setter for id
:Parameters:
id : string
A string uniquely identifying the OMERO.fs Monitor created
by the OMERO.fs Server.
:return: No explicit return value.
"""
#: A string uniquely identifying the OMERO.fs Monitor
self.id = id
def setDirImportWait(self, dirImportWait):
"""
Setter for dirImportWait
:Parameters:
dirImportWait : int
:return: No explicit return value.
"""
self.dirImportWait = dirImportWait
def setThrottleImport(self, throttleImport):
"""
Setter for throttleImport
:Parameters:
throttleImport : int
:return: No explicit return value.
"""
self.throttleImport = throttleImport
def setTimeouts(self, timeToLive, timeToIdle):
"""
Set timeToLive and timeToIdle
:Parameters:
timeToLive : long
timeToIdle : long
:return: No explicit return value.
"""
self.timeToLive = timeToLive
self.timeToIdle = timeToIdle
def setHostAndPort(self, host, port):
"""
Set the host and port from the communicator properties.
"""
self.host = host
self.port = port
def setReaders(self, readers):
"""
Set the readers file from the communicator properties.
"""
self.readers = readers
def setImportArgs(self, importArgs):
"""
Set the importArgs from the communicator properties.
"""
self.importArgs = importArgs
#
# Various trivial helpers
#
def eventRecord(self, category, value):
self.log.info("EVENT_RECORD::%s::%s::%s::%s" % ("Cookie", time.time(), category, value))
def warnAndThrow(self, exc, message, *arguments):
self.log.warn(message, *arguments)
exc.message = (message % arguments)
raise exc
def errAndThrow(self, exc, message, *arguments):
self.log.error(message, *arguments)
exc.message = (message % arguments)
raise exc
class MonitorClientI(monitors.MonitorClient):
"""
Implementation of the MonitorClient.
The interface of the MonitorClient is defined in omerofs.ice and
contains the single callback below.
"""
def __init__(self, dir, communicator, getUsedFiles=as_dictionary, ctx=None,
worker_wait=60, worker_count=1, worker_batch=10):
"""
Intialise the instance variables.
"""
self.log = logging.getLogger("fsclient." + __name__)
self.communicator = communicator
self.master = None
#: Reference back to FSServer.
self.serverProxy = None
self.selfProxy = None
self.dropBoxDir = dir
self.host = ""
self.port = 0
self.dirImportWait = 0
self.throttleImport = 5
self.timeToLive = 0
self.timeToIdle = 0
self.readers = ""
self.importArgs = ""
#: Id
self.id = ''
# Overriding methods to allow for simpler testing
self.getUsedFiles = perf(getUsedFiles)
# Threading primitives
self.worker_wait = worker_wait
self.worker_count = worker_count
self.worker_batch = worker_batch
self.event = get_event()
self.queue = Queue.Queue(0)
self.state = MonitorState(self.event)
self.resources = Resources(stop_event=self.event)
if ctx:
# Primarily used for testing
self.ctx = ctx
else:
self.ctx = ServerContext(
server_id="DropBox", communicator=communicator,
stop_event=self.event)
self.resources.add(self.ctx)
self.workers = [
MonitorWorker(
worker_wait, worker_batch, self.event,
self.queue, self.callback)
for x in range(worker_count)]
for worker in self.workers:
worker.start()
self.eventRecord("Directory", self.dropBoxDir)
@perf
def stop(self):
"""
Shutdown this servant
"""
self.event.set() # Marks everything as stopping
# Shutdown all workers first, otherwise
# there will be contention on the state
workers = self.workers
self.workers = None
if workers:
self.log.info("Joining workers...")
for x in workers:
x.join()
try:
state = self.state
self.state = None
self.log.info("Stopping state...")
if state:
state.stop()
except:
self.log.exception("Error stopping state")
try:
resources = self.resources
self.resources = None
self.log.info("Cleaning up resources state...")
if resources:
resources.cleanup()
except:
self.log.exception("Error cleaning resources")
def __del__(self):
self.stop()
#
# Called by server threads.
#
@remoted
@perf
def fsEventHappened(self, monitorid, eventList, current=None):
"""
Primary monitor client callback.
If new files appear on the watch, the list is sent as an argument.
The id should match for the events to be relevant.
At the moment each file type is treated as a special case. The
number of special cases is likely to explode and so a different
approach is needed. That will be easier with more knowledge of the
different multi-file formats.
:Parameters:
id : string
A string uniquely identifying the OMERO.fs Watch created
by the OMERO.fs Server.
eventList : list<string>
A list of events, in the current implementation this is
a list of strings representing the full path names of new
files.
current
An ICE context, this parameter is required to be present
in an ICE callback.
:return: No explicit return value
"""
# ! Set import to dummy mode for testing purposes.
# self.importFile = self.dummyImportFile
# ! If the above line is not commented out nothing will import.
if self.id != monitorid:
self.warnAndThrow(
omero.ApiUsageException(),
"Unknown fs server id: %s", monitorid)
self.eventRecord("Batch", len(eventList))
for fileInfo in eventList:
fileId = fileInfo.fileId
if not fileId:
self.warnAndThrow(omero.ApiUsageException(), "Empty fieldId")
self.eventRecord(fileInfo.type, fileId)
# Checking name first since it's faster
exName = self.getExperimenterFromPath(fileId)
if exName and self.userExists(exName):
# Creation or modification handled by state/timeout system
if (str(fileInfo.type) == "Create"
or str(fileInfo.type) == "Modify"):
self.queue.put(fileInfo)
else:
self.log.info(
"Event not Create or Modify, presently ignored.")
#
# Called by worker threads.
#
@perf
def callback(self, ids):
try:
self.log.info("Getting filesets on : %s", ids)
fileSets = self.getUsedFiles(list(ids), readers=self.readers)
self.eventRecord("Filesets", str(fileSets))
except:
self.log.exception("Failed to get filesets")
fileSets = None
if fileSets:
self.state.update(
fileSets, self.dirImportWait, self.importFileWrapper)
#
# Called from state callback (timer)
#
def importFileWrapper(self, fileId):
"""
Wrapper method which allows plugging error handling code around
the main call to importFile. In all cases, the key will be removed
on execution.
"""
self.state.clear(fileId)
exName = self.getExperimenterFromPath(fileId)
self.importFile(fileId, exName)
#
# Helpers
#
def getExperimenterFromPath(self, fileId=""):
"""
Extract experimenter name from path. If the experimenter
cannot be extracted, then null will be returned, in which
case no import should take place.
"""
fileId = pathModule.path(fileId)
exName = None
parpath = fileId.parpath(self.dropBoxDir)
if parpath and len(parpath) >= 2:
fileParts = fileId.splitall()
i = -1 * len(parpath)
fileParts = fileParts[i:]
# For .../DropBox/user structure
if len(fileParts) >= 2:
exName = fileParts[0]
# For .../DropBox/u/user structure
# if len(fileParts) >= 3:
# exName = fileParts[1]
if not exName:
self.log.error("File added outside user directories: %s" % fileId)
return exName
def loginUser(self, exName):
"""
Logins in the given user and returns the client
"""
if not self.ctx.hasSession():
self.ctx.newSession()
sf = None
try:
sf = self.ctx.getSession()
except:
self.log.exception("Failed to get sf \n")
if not sf:
self.log.error("No connection")
return None
p = omero.sys.Principal()
p.name = exName
p.group = "user"
p.eventType = "User"
try:
sf.getAdminService().lookupExperimenter(exName)
sess = sf.getSessionService().createSessionWithTimeouts(
p, self.timeToLive, self.timeToIdle)
return sess.uuid.val
except omero.ApiUsageException:
self.log.info("User unknown: %s", exName)
return None
except:
self.log.exception("Unknown exception during loginUser")
return None
def userExists(self, exName):
"""
Tests if the given user exists.
"""
if not self.ctx.hasSession():
self.ctx.newSession()
sf = None
try:
sf = self.ctx.getSession()
except:
self.log.exception("Failed to get sf \n")
if not sf:
self.log.error("No connection")
return False
try:
sf.getAdminService().lookupExperimenter(exName)
return True
except omero.ApiUsageException:
self.log.info("User unknown: %s", exName)
return False
except:
self.log.exception("Unknown exception during loginUser")
return False
@perf
def importFile(self, fileName, exName):
"""
Import file or directory using 'bin/omero importer'
This method is solely responsible for logging the user in,
attempting (possibly multiply) an import, logging and
throwing an exception if necessary.
"""
try:
self.state.appropriateWait(self.throttleImport) # See ticket:5739
key = self.loginUser(exName)
if not key:
self.log.info("File not imported: %s", fileName)
return
self.log.info("Importing %s (session=%s)", fileName, key)
imageId = []
t = create_path("dropbox", "err")
to = create_path("dropbox", "out")
cli = omero.cli.CLI()
cli.loadplugins()
cmd = ["-s", self.host, "-p", str(self.port), "-k", key, "import"]
cmd.extend(
[str("---errs=%s" % t),
str("---file=%s" % to), "--", "--agent=dropbox"])
cmd.extend(shlex.split(self.importArgs))
cmd.append(fileName)
logging.debug("cli.invoke(%s)" % cmd)
cli.invoke(cmd)
retCode = cli.rv
if retCode == 0:
self.log.info(
"Import of %s completed (session=%s)", fileName, key)
if to.exists():
f = open(str(to), "r")
lines = f.readlines()
f.close()
if len(lines) > 0:
for line in lines:
imageId.append(line.strip())
else:
self.log.error("No lines in output file. No image ID.")
else:
self.log.error("%s not found !" % to)
else:
self.log.error(
"Import of %s failed=%s (session=%s)",
fileName, str(retCode), key)
self.log.error(
"***** start of output from importer-cli to stderr *****")
if t.exists():
f = open(str(t), "r")
lines = f.readlines()
f.close()
for line in lines:
self.log.error(line.strip())
else:
self.log.error("%s not found !" % t)
self.log.error("***** end of output from importer-cli *****")
finally:
remove_path(t)
remove_path(to)
return imageId
#
# Setters
#
def dummyImportFile(self, fileName, exName):
"""
Log a potential import for test purposes
"""
self.log.info(
"***DUMMY IMPORT*** Would have tried to import: %s ", fileName)
def setMaster(self, master):
"""
Setter for FSDropBox
:Parameters:
master : DropBox
DropBox Server
:return: No explicit return value.
"""
self.master = master
def setServerProxy(self, serverProxy):
"""
Setter for serverProxy
:Parameters:
serverProxy : monitors.MonitorServerPrx
proxy to remote server object
:return: No explicit return value.
"""
self.serverProxy = serverProxy
def setSelfProxy(self, selfProxy):
"""
Setter for serverProxy
:Parameters:
selfProxy : monitors.MonitorClientPrx
proxy to this client object
:return: No explicit return value.
"""
self.selfProxy = selfProxy
def setId(self, id):
"""
Setter for id
:Parameters:
id : string
A string uniquely identifying the OMERO.fs Monitor created
by the OMERO.fs Server.
:return: No explicit return value.
"""
#: A string uniquely identifying the OMERO.fs Monitor
self.id = id
def setDirImportWait(self, dirImportWait):
"""
Setter for dirImportWait
:Parameters:
dirImportWait : int
:return: No explicit return value.
"""
self.dirImportWait = dirImportWait
def setThrottleImport(self, throttleImport):
"""
Setter for throttleImport
:Parameters:
throttleImport : int
:return: No explicit return value.
"""
self.throttleImport = throttleImport
def setTimeouts(self, timeToLive, timeToIdle):
"""
Set timeToLive and timeToIdle
:Parameters:
timeToLive : long
timeToIdle : long
:return: No explicit return value.
"""
self.timeToLive = timeToLive
self.timeToIdle = timeToIdle
def setHostAndPort(self, host, port):
"""
Set the host and port from the communicator properties.
"""
self.host = host
self.port = port
def setReaders(self, readers):
"""
Set the readers file from the communicator properties.
"""
self.readers = readers
def setImportArgs(self, importArgs):
"""
Set the importArgs from the communicator properties.
"""
self.importArgs = importArgs
#
# Various trivial helpers
#
def eventRecord(self, category, value):
self.log.info("EVENT_RECORD::%s::%s::%s::%s" %
("Cookie", time.time(), category, value))
def warnAndThrow(self, exc, message, *arguments):
self.log.warn(message, *arguments)
exc.message = (message % arguments)
raise exc
def errAndThrow(self, exc, message, *arguments):
self.log.error(message, *arguments)
exc.message = (message % arguments)
raise exc
