def __init__(self, dir, communicator, getUsedFiles=as_dictionary, ctx=None, worker_wait=60, worker_count=1, worker_batch=10): """ Intialise the instance variables. """ self.log = logging.getLogger("fsclient." + __name__) self.communicator = communicator self.master = None #: Reference back to FSServer. self.serverProxy = None self.selfProxy = None self.dropBoxDir = dir self.host = "" self.port = 0 self.dirImportWait = 0 self.throttleImport = 5 self.timeToLive = 0 self.timeToIdle = 0 self.readers = "" self.importArgs = "" #: Id self.id = '' # Overriding methods to allow for simpler testing self.getUsedFiles = perf(getUsedFiles) # Threading primitives self.worker_wait = worker_wait self.worker_count = worker_count self.worker_batch = worker_batch self.event = get_event() self.queue = Queue.Queue(0) self.state = MonitorState(self.event) self.resources = Resources(stop_event=self.event) if ctx: # Primarily used for testing self.ctx = ctx else: self.ctx = ServerContext( server_id="DropBox", communicator=communicator, stop_event=self.event) self.resources.add(self.ctx) self.workers = [ MonitorWorker( worker_wait, worker_batch, self.event, self.queue, self.callback) for x in range(worker_count)] for worker in self.workers: worker.start() self.eventRecord("Directory", self.dropBoxDir)
class MonitorClientI(monitors.MonitorClient): """ Implementation of the MonitorClient. The interface of the MonitorClient is defined in omerofs.ice and contains the single callback below. """ def __init__(self, dir, communicator, getUsedFiles = as_dictionary, ctx = None,\ worker_wait = 60, worker_count = 1, worker_batch = 10): """ Intialise the instance variables. """ self.log = logging.getLogger("fsclient."+__name__) self.communicator = communicator self.master = None #: Reference back to FSServer. self.serverProxy = None self.selfProxy = None self.dropBoxDir = dir self.host = "" self.port = 0 self.dirImportWait = 0 self.throttleImport = 5 self.timeToLive = 0 self.timeToIdle = 0 self.readers = "" self.importArgs = "" #: Id self.id = '' # Overriding methods to allow for simpler testing self.getUsedFiles = perf(getUsedFiles) # Threading primitives self.worker_wait = worker_wait self.worker_count = worker_count self.worker_batch = worker_batch self.event = get_event() self.queue = Queue.Queue(0) self.state = MonitorState(self.event) self.resources = Resources(stop_event = self.event) if ctx: # Primarily used for testing self.ctx = ctx else: self.ctx = ServerContext(server_id = "DropBox", communicator = communicator, stop_event = self.event) self.resources.add(self.ctx) self.workers = [MonitorWorker(worker_wait, worker_batch, self.event, self.queue, self.callback) for x in range(worker_count)] for worker in self.workers: worker.start() self.eventRecord("Directory", self.dropBoxDir) @perf def stop(self): """ Shutdown this servant """ self.event.set() # Marks everything as stopping # Shutdown all workers first, otherwise # there will be contention on the state workers = self.workers self.workers = None if workers: self.log.info("Joining workers...") for x in workers: x.join() try: state = self.state self.state = None self.log.info("Stopping state...") if state: state.stop() except: self.log.exception("Error stopping state") try: resources = self.resources self.resources = None self.log.info("Cleaning up resources state...") if resources: resources.cleanup() except: self.log.exception("Error cleaning resources") def __del__(self): self.stop() # # Called by server threads. # @remoted @perf def fsEventHappened(self, monitorid, eventList, current=None): """ Primary monitor client callback. If new files appear on the watch, the list is sent as an argument. The id should match for the events to be relevant. At the moment each file type is treated as a special case. The number of special cases is likely to explode and so a different approach is needed. That will be easier with more knowledge of the different multi-file formats. :Parameters: id : string A string uniquely identifying the OMERO.fs Watch created by the OMERO.fs Server. eventList : list<string> A list of events, in the current implementation this is a list of strings representing the full path names of new files. current An ICE context, this parameter is required to be present in an ICE callback. :return: No explicit return value """ # ############## ! Set import to dummy mode for testing purposes. # self.importFile = self.dummyImportFile # ############## ! If the above line is not commented out nothing will import. if self.id != monitorid: self.warnAndThrow(omero.ApiUsageException(), "Unknown fs server id: %s", monitorid) self.eventRecord("Batch", len(eventList)) for fileInfo in eventList: fileId = fileInfo.fileId if not fileId: self.warnAndThrow(omero.ApiUsageException(), "Empty fieldId") self.eventRecord(fileInfo.type, fileId) # Checking name first since it's faster exName = self.getExperimenterFromPath(fileId) if exName and self.userExists(exName): # Creation or modification handled by state/timeout system if str(fileInfo.type) == "Create" or str(fileInfo.type) == "Modify": self.queue.put(fileInfo) else: self.log.info("Event not Create or Modify, presently ignored.") # # Called by worker threads. # @perf def callback(self, ids): try: self.log.info("Getting filesets on : %s", ids) fileSets = self.getUsedFiles(list(ids), readers=self.readers) self.eventRecord("Filesets", str(fileSets)) except: self.log.exception("Failed to get filesets") fileSets = None if fileSets: self.state.update(fileSets, self.dirImportWait, self.importFileWrapper) # # Called from state callback (timer) # def importFileWrapper(self, fileId): """ Wrapper method which allows plugging error handling code around the main call to importFile. In all cases, the key will be removed on execution. """ self.state.clear(fileId) exName = self.getExperimenterFromPath(fileId) self.importFile(fileId, exName) # # Helpers # def getExperimenterFromPath(self, fileId=""): """ Extract experimenter name from path. If the experimenter cannot be extracted, then null will be returned, in which case no import should take place. """ fileId = pathModule.path(fileId) exName = None parpath = fileId.parpath(self.dropBoxDir) if parpath and len(parpath) >= 2: fileParts = fileId.splitall() i = -1 * len(parpath) fileParts = fileParts[i:] # For .../DropBox/user structure if len(fileParts) >= 2: exName = fileParts[0] # For .../DropBox/u/user structure #if len(fileParts) >= 3: # exName = fileParts[1] if not exName: self.log.error("File added outside user directories: %s" % fileId) return exName def loginUser(self, exName): """ Logins in the given user and returns the client """ if not self.ctx.hasSession(): self.ctx.newSession() sf = None try: sf = self.ctx.getSession() except: self.log.exception("Failed to get sf \n") if not sf: self.log.error("No connection") return None p = omero.sys.Principal() p.name = exName p.group = "user" p.eventType = "User" try: exp = sf.getAdminService().lookupExperimenter(exName) sess = sf.getSessionService().createSessionWithTimeouts(p, self.timeToLive, self.timeToIdle) return sess.uuid.val except omero.ApiUsageException: self.log.info("User unknown: %s", exName) return None except: self.log.exception("Unknown exception during loginUser") return None def userExists(self, exName): """ Tests if the given user exists. """ if not self.ctx.hasSession(): self.ctx.newSession() sf = None try: sf = self.ctx.getSession() except: self.log.exception("Failed to get sf \n") if not sf: self.log.error("No connection") return False try: exp = sf.getAdminService().lookupExperimenter(exName) return True except omero.ApiUsageException: self.log.info("User unknown: %s", exName) return False except: self.log.exception("Unknown exception during loginUser") return False @perf def importFile(self, fileName, exName): """ Import file or directory using 'bin/omero importer' This method is solely responsible for logging the user in, attempting (possibly multiply) an import, logging and throwing an exception if necessary. """ key = self.loginUser(exName) if not key: self.log.info("File not imported: %s", fileName) return try: self.state.appropriateWait(self.throttleImport) # See ticket:5739 self.log.info("Importing %s (session=%s)", fileName, key) imageId = [] t = create_path("dropbox", "err") to = create_path("dropbox", "out") cli = omero.cli.CLI() cli.loadplugins() cmd = ["-s", self.host, "-p", str(self.port), "-k", key, "import"] cmd.extend([str("---errs=%s"%t), str("---file=%s"%to), "--", "--agent=dropbox"]) cmd.extend(shlex.split(self.importArgs)) cmd.append(fileName) logging.debug("cli.invoke(%s)" % cmd) cli.invoke(cmd) retCode = cli.rv if retCode == 0: self.log.info("Import of %s completed (session=%s)", fileName, key) if to.exists(): f = open(str(to),"r") lines = f.readlines() f.close() if len(lines) > 0: for line in lines: imageId.append(line.strip()) else: self.log.error("No lines in output file. No image ID.") else: self.log.error("%s not found !" % to) else: self.log.error("Import of %s failed=%s (session=%s)", fileName, str(retCode), key) self.log.error("***** start of output from importer-cli to stderr *****") if t.exists(): f = open(str(t),"r") lines = f.readlines() f.close() for line in lines: self.log.error(line.strip()) else: self.log.error("%s not found !" % t) self.log.error("***** end of output from importer-cli *****") finally: remove_path(t) remove_path(to) return imageId # # Setters # def dummyImportFile(self, fileName, exName): """ Log a potential import for test purposes """ self.log.info("***DUMMY IMPORT*** Would have tried to import: %s ", fileName) def setMaster(self, master): """ Setter for FSDropBox :Parameters: master : DropBox DropBox Server :return: No explicit return value. """ self.master = master def setServerProxy(self, serverProxy): """ Setter for serverProxy :Parameters: serverProxy : monitors.MonitorServerPrx proxy to remote server object :return: No explicit return value. """ self.serverProxy = serverProxy def setSelfProxy(self, selfProxy): """ Setter for serverProxy :Parameters: selfProxy : monitors.MonitorClientPrx proxy to this client object :return: No explicit return value. """ self.selfProxy = selfProxy def setId(self, id): """ Setter for id :Parameters: id : string A string uniquely identifying the OMERO.fs Monitor created by the OMERO.fs Server. :return: No explicit return value. """ #: A string uniquely identifying the OMERO.fs Monitor self.id = id def setDirImportWait(self, dirImportWait): """ Setter for dirImportWait :Parameters: dirImportWait : int :return: No explicit return value. """ self.dirImportWait = dirImportWait def setThrottleImport(self, throttleImport): """ Setter for throttleImport :Parameters: throttleImport : int :return: No explicit return value. """ self.throttleImport = throttleImport def setTimeouts(self, timeToLive, timeToIdle): """ Set timeToLive and timeToIdle :Parameters: timeToLive : long timeToIdle : long :return: No explicit return value. """ self.timeToLive = timeToLive self.timeToIdle = timeToIdle def setHostAndPort(self, host, port): """ Set the host and port from the communicator properties. """ self.host = host self.port = port def setReaders(self, readers): """ Set the readers file from the communicator properties. """ self.readers = readers def setImportArgs(self, importArgs): """ Set the importArgs from the communicator properties. """ self.importArgs = importArgs # # Various trivial helpers # def eventRecord(self, category, value): self.log.info("EVENT_RECORD::%s::%s::%s::%s" % ("Cookie", time.time(), category, value)) def warnAndThrow(self, exc, message, *arguments): self.log.warn(message, *arguments) exc.message = (message % arguments) raise exc def errAndThrow(self, exc, message, *arguments): self.log.error(message, *arguments) exc.message = (message % arguments) raise exc
class MonitorClientI(monitors.MonitorClient): """ Implementation of the MonitorClient. The interface of the MonitorClient is defined in omerofs.ice and contains the single callback below. """ def __init__(self, dir, communicator, getUsedFiles=as_dictionary, ctx=None, worker_wait=60, worker_count=1, worker_batch=10): """ Intialise the instance variables. """ self.log = logging.getLogger("fsclient." + __name__) self.communicator = communicator self.master = None #: Reference back to FSServer. self.serverProxy = None self.selfProxy = None self.dropBoxDir = dir self.host = "" self.port = 0 self.dirImportWait = 0 self.throttleImport = 5 self.timeToLive = 0 self.timeToIdle = 0 self.readers = "" self.importArgs = "" #: Id self.id = '' # Overriding methods to allow for simpler testing self.getUsedFiles = perf(getUsedFiles) # Threading primitives self.worker_wait = worker_wait self.worker_count = worker_count self.worker_batch = worker_batch self.event = get_event() self.queue = Queue.Queue(0) self.state = MonitorState(self.event) self.resources = Resources(stop_event=self.event) if ctx: # Primarily used for testing self.ctx = ctx else: self.ctx = ServerContext( server_id="DropBox", communicator=communicator, stop_event=self.event) self.resources.add(self.ctx) self.workers = [ MonitorWorker( worker_wait, worker_batch, self.event, self.queue, self.callback) for x in range(worker_count)] for worker in self.workers: worker.start() self.eventRecord("Directory", self.dropBoxDir) @perf def stop(self): """ Shutdown this servant """ self.event.set() # Marks everything as stopping # Shutdown all workers first, otherwise # there will be contention on the state workers = self.workers self.workers = None if workers: self.log.info("Joining workers...") for x in workers: x.join() try: state = self.state self.state = None self.log.info("Stopping state...") if state: state.stop() except: self.log.exception("Error stopping state") try: resources = self.resources self.resources = None self.log.info("Cleaning up resources state...") if resources: resources.cleanup() except: self.log.exception("Error cleaning resources") def __del__(self): self.stop() # # Called by server threads. # @remoted @perf def fsEventHappened(self, monitorid, eventList, current=None): """ Primary monitor client callback. If new files appear on the watch, the list is sent as an argument. The id should match for the events to be relevant. At the moment each file type is treated as a special case. The number of special cases is likely to explode and so a different approach is needed. That will be easier with more knowledge of the different multi-file formats. :Parameters: id : string A string uniquely identifying the OMERO.fs Watch created by the OMERO.fs Server. eventList : list<string> A list of events, in the current implementation this is a list of strings representing the full path names of new files. current An ICE context, this parameter is required to be present in an ICE callback. :return: No explicit return value """ # ! Set import to dummy mode for testing purposes. # self.importFile = self.dummyImportFile # ! If the above line is not commented out nothing will import. if self.id != monitorid: self.warnAndThrow( omero.ApiUsageException(), "Unknown fs server id: %s", monitorid) self.eventRecord("Batch", len(eventList)) for fileInfo in eventList: fileId = fileInfo.fileId if not fileId: self.warnAndThrow(omero.ApiUsageException(), "Empty fieldId") self.eventRecord(fileInfo.type, fileId) # Checking name first since it's faster exName = self.getExperimenterFromPath(fileId) if exName and self.userExists(exName): # Creation or modification handled by state/timeout system if (str(fileInfo.type) == "Create" or str(fileInfo.type) == "Modify"): self.queue.put(fileInfo) else: self.log.info( "Event not Create or Modify, presently ignored.") # # Called by worker threads. # @perf def callback(self, ids): try: self.log.info("Getting filesets on : %s", ids) fileSets = self.getUsedFiles(list(ids), readers=self.readers) self.eventRecord("Filesets", str(fileSets)) except: self.log.exception("Failed to get filesets") fileSets = None if fileSets: self.state.update( fileSets, self.dirImportWait, self.importFileWrapper) # # Called from state callback (timer) # def importFileWrapper(self, fileId): """ Wrapper method which allows plugging error handling code around the main call to importFile. In all cases, the key will be removed on execution. """ self.state.clear(fileId) exName = self.getExperimenterFromPath(fileId) self.importFile(fileId, exName) # # Helpers # def getExperimenterFromPath(self, fileId=""): """ Extract experimenter name from path. If the experimenter cannot be extracted, then null will be returned, in which case no import should take place. """ fileId = pathModule.path(fileId) exName = None parpath = fileId.parpath(self.dropBoxDir) if parpath and len(parpath) >= 2: fileParts = fileId.splitall() i = -1 * len(parpath) fileParts = fileParts[i:] # For .../DropBox/user structure if len(fileParts) >= 2: exName = fileParts[0] # For .../DropBox/u/user structure # if len(fileParts) >= 3: # exName = fileParts[1] if not exName: self.log.error("File added outside user directories: %s" % fileId) return exName def loginUser(self, exName): """ Logins in the given user and returns the client """ if not self.ctx.hasSession(): self.ctx.newSession() sf = None try: sf = self.ctx.getSession() except: self.log.exception("Failed to get sf \n") if not sf: self.log.error("No connection") return None p = omero.sys.Principal() p.name = exName p.group = "user" p.eventType = "User" try: sf.getAdminService().lookupExperimenter(exName) sess = sf.getSessionService().createSessionWithTimeouts( p, self.timeToLive, self.timeToIdle) return sess.uuid.val except omero.ApiUsageException: self.log.info("User unknown: %s", exName) return None except: self.log.exception("Unknown exception during loginUser") return None def userExists(self, exName): """ Tests if the given user exists. """ if not self.ctx.hasSession(): self.ctx.newSession() sf = None try: sf = self.ctx.getSession() except: self.log.exception("Failed to get sf \n") if not sf: self.log.error("No connection") return False try: sf.getAdminService().lookupExperimenter(exName) return True except omero.ApiUsageException: self.log.info("User unknown: %s", exName) return False except: self.log.exception("Unknown exception during loginUser") return False @perf def importFile(self, fileName, exName): """ Import file or directory using 'bin/omero importer' This method is solely responsible for logging the user in, attempting (possibly multiply) an import, logging and throwing an exception if necessary. """ try: self.state.appropriateWait(self.throttleImport) # See ticket:5739 key = self.loginUser(exName) if not key: self.log.info("File not imported: %s", fileName) return self.log.info("Importing %s (session=%s)", fileName, key) imageId = [] t = create_path("dropbox", "err") to = create_path("dropbox", "out") cli = omero.cli.CLI() cli.loadplugins() cmd = ["-s", self.host, "-p", str(self.port), "-k", key, "import"] cmd.extend( [str("---errs=%s" % t), str("---file=%s" % to), "--", "--agent=dropbox"]) cmd.extend(shlex.split(self.importArgs)) cmd.append(fileName) logging.debug("cli.invoke(%s)" % cmd) cli.invoke(cmd) retCode = cli.rv if retCode == 0: self.log.info( "Import of %s completed (session=%s)", fileName, key) if to.exists(): f = open(str(to), "r") lines = f.readlines() f.close() if len(lines) > 0: for line in lines: imageId.append(line.strip()) else: self.log.error("No lines in output file. No image ID.") else: self.log.error("%s not found !" % to) else: self.log.error( "Import of %s failed=%s (session=%s)", fileName, str(retCode), key) self.log.error( "***** start of output from importer-cli to stderr *****") if t.exists(): f = open(str(t), "r") lines = f.readlines() f.close() for line in lines: self.log.error(line.strip()) else: self.log.error("%s not found !" % t) self.log.error("***** end of output from importer-cli *****") finally: remove_path(t) remove_path(to) return imageId # # Setters # def dummyImportFile(self, fileName, exName): """ Log a potential import for test purposes """ self.log.info( "***DUMMY IMPORT*** Would have tried to import: %s ", fileName) def setMaster(self, master): """ Setter for FSDropBox :Parameters: master : DropBox DropBox Server :return: No explicit return value. """ self.master = master def setServerProxy(self, serverProxy): """ Setter for serverProxy :Parameters: serverProxy : monitors.MonitorServerPrx proxy to remote server object :return: No explicit return value. """ self.serverProxy = serverProxy def setSelfProxy(self, selfProxy): """ Setter for serverProxy :Parameters: selfProxy : monitors.MonitorClientPrx proxy to this client object :return: No explicit return value. """ self.selfProxy = selfProxy def setId(self, id): """ Setter for id :Parameters: id : string A string uniquely identifying the OMERO.fs Monitor created by the OMERO.fs Server. :return: No explicit return value. """ #: A string uniquely identifying the OMERO.fs Monitor self.id = id def setDirImportWait(self, dirImportWait): """ Setter for dirImportWait :Parameters: dirImportWait : int :return: No explicit return value. """ self.dirImportWait = dirImportWait def setThrottleImport(self, throttleImport): """ Setter for throttleImport :Parameters: throttleImport : int :return: No explicit return value. """ self.throttleImport = throttleImport def setTimeouts(self, timeToLive, timeToIdle): """ Set timeToLive and timeToIdle :Parameters: timeToLive : long timeToIdle : long :return: No explicit return value. """ self.timeToLive = timeToLive self.timeToIdle = timeToIdle def setHostAndPort(self, host, port): """ Set the host and port from the communicator properties. """ self.host = host self.port = port def setReaders(self, readers): """ Set the readers file from the communicator properties. """ self.readers = readers def setImportArgs(self, importArgs): """ Set the importArgs from the communicator properties. """ self.importArgs = importArgs # # Various trivial helpers # def eventRecord(self, category, value): self.log.info("EVENT_RECORD::%s::%s::%s::%s" % ("Cookie", time.time(), category, value)) def warnAndThrow(self, exc, message, *arguments): self.log.warn(message, *arguments) exc.message = (message % arguments) raise exc def errAndThrow(self, exc, message, *arguments): self.log.error(message, *arguments) exc.message = (message % arguments) raise exc