class FsRadar: def __init__(self, dir_filter, observer): self.inotify = INotify() self.watch_flags = flags.CREATE | flags.DELETE | flags.MODIFY | flags.DELETE_SELF self.watch_flags = masks.ALL_EVENTS self.watch_flags = \ flags.CREATE | \ flags.DELETE | \ flags.DELETE_SELF | \ flags.CLOSE_WRITE | \ flags.MOVE_SELF | \ flags.MOVED_FROM | \ flags.MOVED_TO | \ flags.EXCL_UNLINK self.wds = {} self.dir_filter = dir_filter self.observer = observer def add_watch(self, path): if not ((self.watch_flags & flags.ONLYDIR) and not os.path.isdir(path)): wd = self.inotify.add_watch(path, self.watch_flags) self.wds[wd] = path logger.debug('Watch %s', important(path)) def rm_watch(self, wd): logger.debug('Stop Watching %s', important(self.wds[wd])) self.inotify.rm_watch(wd) self.wds.pop(wd) def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): self.close() def close(self): logger.debug('Close inotify descriptor') return self.inotify.close() def on_watch_event(self, event): MASK_NEW_DIR = flags.CREATE | flags.ISDIR if logging.getLogger().isEnabledFor(logging.DEBUG): logger.debug('New event: %r', event) for flag in flags.from_mask(event.mask): logger.debug('-> flag: %s', flag) if MASK_NEW_DIR == MASK_NEW_DIR & event.mask: new_dir_path = join(self.wds[event.wd], event.name) self.on_new_dir(new_dir_path) elif flags.CLOSE_WRITE & event.mask and event.name: # we are watching a directory and a file inside of it has been touched logger.debug('Watching dir, file touched') self.on_file_write(join(self.wds[event.wd], event.name)) elif flags.CLOSE_WRITE & event.mask and not event.name: # we are watching a file logger.debug('Watching file, file touched') self.on_file_write(self.wds[event.wd]) elif flags.IGNORED & event.mask: # inotify_rm_watch was called automatically # (file/directory removed/unmounted) path = self.wds[event.wd] self.wds.pop(event.wd) self.on_file_gone(path) def on_new_dir(self, path): if self.dir_filter(path): self.add_watch(path) # If files have been added immediately to the directory we # missed the events, so we emit them artificially (with # the risk of having some repeated events) for fName in os.listdir(path): self.on_file_write(join(new_dir_path, fName)) def on_file_write(self, path): '''A write /directory at `path` was either unlinked, moved or unmounted''' self.observer.notify(FsRadarEvent.FILE_MATCH, path) def on_file_gone(self, path): '''The file/directory at `path` was either unlinked, moved or unmounted''' self.observer.notify(FsRadarEvent.FILE_GONE, path) def run_forever(self): while True: for event in self.inotify.read(read_delay=30, timeout=2000): self.on_watch_event(event)
def loop_inotify(self, mail_delta): directory = self.file_consumer.consume inotify = INotify() inotify.add_watch(directory, flags.CLOSE_WRITE | flags.MOVED_TO) # Run initial mail fetch and consume all currently existing documents self.loop_step(mail_delta) next_mail_time = self.mail_fetcher.last_checked + mail_delta while True: # Consume documents until next_mail_time while True: delta = next_mail_time - time.time() if delta > 0: for event in inotify.read(timeout=delta): file = os.path.join(directory, event.name) if os.path.isfile(file): self.file_consumer.try_consume_file(file) else: self.logger.warning( "Skipping %s as it is not a file", file) else: break self.mail_fetcher.pull() next_mail_time = self.mail_fetcher.last_checked + mail_delta
def _main(): watch_dir = "/tmp/watch_tree" create_thread = threading.Thread(target=create_test_files, args=(watch_dir,)) try: os.mkdir(watch_dir) except OSError: pass inotify = INotify() watch_flags = (flags.CREATE | flags.DELETE | flags.MODIFY | flags.DELETE_SELF | flags.ISDIR) wd_to_path = {} wd = inotify.add_watch(watch_dir, watch_flags) wd_to_path[wd] = watch_dir print("wd_to_path: ", wd_to_path) create_thread.start() while True: for event in inotify.read(): path = wd_to_path[event.wd] event_type = flags.from_mask(event.mask) # remember dir name if flags.ISDIR in event_type and flags.CREATE in event_type: new_dir = os.path.join(path, event.name) wd = inotify.add_watch(new_dir, watch_flags) wd_to_path[wd] = new_dir print("PATH=[{}] FILENAME=[{}] EVENT_TYPES={}" .format(path, event.name, event_type))
def _main(): watch_dir = "/tmp/watch_tree" try: for path in watch_dir: os.mkdir(path) except OSError: pass inotify = INotify() watch_flags = flags.CREATE | flags.DELETE | flags.MODIFY | flags.DELETE_SELF wd_to_path = {} wd = inotify.add_watch(watch_dir, watch_flags) wd_to_path[wd] = watch_dir print("wd_to_path: ", wd_to_path) with open(os.path.join(watch_dir, "test_file"), "w"): pass while True: for event in inotify.read(): path = wd_to_path[event.wd] event_type = flags.from_mask(event.mask) print("PATH=[{}] FILENAME=[{}] EVENT_TYPES={}".format( path, event.name, event_type))
def __init__(self, path): self.path = path fl = flags.MODIFY self.i = INotify() self.watch = self.i.add_watch(self.path, fl) with open(self.path, "r") as f: self.content = f.read()
def run(self, inotify: INotify, store: OrderedDict, timeout: int): for file_path in list(store): try: # for now only monitor modification of files logger.info("adding %s to watch list" % file_path) wd = inotify.add_watch(file_path, flags.MODIFY) watches[wd] = file_path except FileNotFoundError: pass # print("watch list %s " % watches) while self._running: for event in inotify.read(timeout=timeout): # print(str(event)) for flag in flags.from_mask(event.mask): # we only watch files for modification events if flag == flags.MODIFY: # only if the event is for a file that we are watching for if event.wd in watches: file_path = watches[event.wd] logger.debug("modify event received for %s " % file_path) store[ file_path] = FStatCache.get_file_stats_using_stat( file_path)
def __init__(self): super().__init__() self.inotify = INotify() self.qnotifier = QSocketNotifier(self.inotify.fd, QSocketNotifier.Read) self.qnotifier.activated.connect(self._on_activated) self.wd = None
class TickNotifierOrigin(): def __init__(self, symbol, binary_id, id): self.__set_config(symbol, binary_id, id) def __set_config(self, symbol, binary_id, id): self.binary_id = binary_id self.id = id self.tick_dir = local_settings["inotify_config"]["default_dir"] + symbol self.tick_file_path = self.tick_dir + "/" + local_settings[ "inotify_config"]["default_name"] self.inotify = INotify() watch_flags = flags.CREATE | flags.DELETE | flags.MODIFY | flags.DELETE_SELF self.inotify.add_watch(self.tick_dir, watch_flags) os.chdir(self.tick_dir) self.tick_data = None self.symbol = symbol def run(self): event = self.inotify.read()[0] for flag in flags.from_mask(event.mask): if str(flag) == "flags.MODIFY": with open(self.tick_file_path) as json_file: self.tick_data = json.load(json_file) return True def get_tick_data(self): return { "id": self.id, "binary_id": self.binary_id, "symbol": self.symbol, "data": self.tick_data }
def init_inotify(self): self.inotify = INotify() self.dirs_array = self.discover_tree(self.watched_dir) for dir in self.dirs_array: self.watched_name[self.inotify.add_watch(dir, self.watch_flags)] = dir print(str(self.watched_name))
def __init__(self): """Summary """ self.inotify = INotify() self.watched_flags = flags.CREATE | flags.MOVED_TO | flags.ISDIR self.directories = {} self.watch_descriptors = {} self.users = {} self.uploaders = [GCSUploader(), S3Uploader()] for user, _ in config.PROJECT_CONFIG["USERS"].items(): watch_descriptor = self.inotify.add_watch( os.path.join(config.APP_LANDING_DIR, user, 'ingest'), self.watched_flags) self.directories[watch_descriptor] = os.path.join( config.APP_LANDING_DIR, user, 'ingest') self.users[watch_descriptor] = user self.watch_descriptors[ self.directories[watch_descriptor]] = watch_descriptor logger.info("Watching ingestion folder") while True: events = self.inotify.read(read_delay=1000) all_events = self.get_all_events(events) with ThreadPoolExecutor(max_workers=None) as executor: for event in all_events: executor.submit( upload_file, self.uploaders, os.path.join(self.directories[event.wd], event.name))
def __init__(self, parent: Optional[QObject] = None) -> None: super().__init__(parent) self.inotify = INotify() self.qnotifier = QSocketNotifier(self.inotify.fd, QSocketNotifier.Read) self.qnotifier.activated.connect(self._on_activated) self.wd = None
class FileChangeWatcher(object): """Monitors for changes in a given file.""" def __init__(self, filepath): self.__path, self.__name = filepath.rsplit("/", 1) self.__inotify = INotify() watch_flags = \ Flags.MODIFY \ | Flags.CLOSE_WRITE \ | Flags.MOVED_TO \ | Flags.EXCL_UNLINK self.__wd = self.__inotify.add_watch(self.__path, watch_flags) def close(self): self.__inotify.rm_watch(self.__wd) @property def changed(self): modified = saved = False events = self.__inotify.read() events = (e for e in events if e.name == self.__name) flags = (f for e in events for f in Flags.from_mask(e.mask)) for flag in flags: if flag == Flags.MODIFY: modified = True elif flag == Flags.CLOSE_WRITE: saved = True elif flag == Flags.MOVED_TO: modified = saved = True return modified and saved
def history_watcher(shutdown_event, session_start_event, session_stop_event): while (not (shutdown_event.isSet())): session_start_event.wait() inotify = INotify() watch_flags = flags.MODIFY shutil.copy(HISTORY_FILE, HISTORY_FILE + "_clj") wd = inotify.add_watch(HISTORY_FILE, watch_flags) print("IDK") while (not (session_stop_event.isSet())): print("yada") for event in inotify.read(): dp = subprocess.Popen(['diff', HISTORY_FILE, HISTORY_FILE + '_clj'], \ stdout=subprocess.PIPE) gp = subprocess.Popen(['grep', 'cmd'], \ stdin=dp.stdout, \ stdout=subprocess.PIPE) dp.stdout.close() r_stdout, r_stderr = gp.communicate() cmd_ln = r_stdout.decode().split('cmd: ')[1] print(cmd_ln) write_journal((("___________________________________________________\n"), \ (getcwd() + "> " + cmd_ln), \ (datetime.now().strftime("%m/%d/%Y %H:%M:%S") + '\n\n'))) shutil.copy(HISTORY_FILE, HISTORY_FILE + "_clj") sleep(0.1)
def __set_config(self, pool_name): self.data = None self.queue_pool_dir = self.main_path + pool_name self.inotify = INotify() # watch_flags = flags.CREATE watch_flags = flags.CREATE | flags.MODIFY self.inotify.add_watch(self.queue_pool_dir, watch_flags) os.chdir(self.queue_pool_dir)
def main(): inotify = INotify() watch_flags = flags.CREATE | flags.DELETE | flags.MODIFY | flags.DELETE_SELF wd = inotify.add_watch('/tmp', watch_flags) for event in inotify.read(): print(event) for flag in flags.from_mask(event.mask): print(' ' + str(flag))
def get_inotify(): inotify = INotify() watch_flags = masks.ALL_EVENTS logger.debug("inotify flags: %s", watch_flags) for path in CONFIG['MOUNTS']: logger.info("Watching path %s", path) wd = inotify.add_watch(path, watch_flags) return inotify
class Watcher: # pylint: disable=too-few-public-methods """Watcher class. Provides `watch` method to execute callback when event happens""" def __init__(self, callback): self._inotify = None self._thread = None self._watch_descriptors = [] self._callback = callback def watch(self, directories): """ Creates a thread to watch for a single fs event. Executes callback once when any monitored event happens. Removes existing watchers on initialization, so only one watching thread exists at a time. Keyword arguments: directories -- directories to watch callback -- function to execute on event """ if self._inotify: for descriptor in self._watch_descriptors: try: self._inotify.rm_watch(descriptor) except OSError: pass self._watch_descriptors = [] self._inotify.close() self._thread = Thread(target=self._notify_changes, args=(directories, )) self._thread.start() def get_notification_thread(self): """Returns notification thread""" return self._thread def _notify_changes(self, directories): """ Registers notification on FS events. Keyword arguments: directory -- directory to watch callback -- function to execute on event """ self._inotify = INotify() watch_flags = flags.CREATE | flags.DELETE | flags.MODIFY | flags.MOVED_FROM | \ flags.MOVED_TO | flags.DELETE_SELF for directory in directories: if exists(directory): self._watch_descriptors.append( self._inotify.add_watch(directory, watch_flags)) # wait for event try: self._inotify.read() self._callback() except ValueError: # do not care if the watcher has been removed / thread killed pass
def watcher(directory): inotify = INotify() watch_flags = flags.CREATE | flags.MOVED_TO # only watch for files created, copied or moved inotify.add_watch(directory, watch_flags) WAIT = 1000 while True: for event in inotify.read(WAIT): filename = event.name yield os.path.join(directory, filename)
class INotifyWait(EventEmitter): IN_ACCESS = flags.ACCESS IN_MODIFY = flags.MODIFY IN_CLOSE_WRITE = flags.CLOSE_WRITE IN_CLOSE_NOWRITE = flags.CLOSE_NOWRITE IN_OPEN = flags.OPEN IN_MOVED_FROM = flags.MOVED_FROM IN_MOVED_TO = flags.MOVED_TO IN_CREATE = flags.CREATE IN_DELETE = flags.DELETE IN_DELETE_SELF = flags.DELETE_SELF IN_MOVE_SELF = flags.MOVE_SELF IN_UNMOUNT = flags.UNMOUNT IN_CLOSE = flags.CLOSE_WRITE | flags.CLOSE_NOWRITE IN_MOVE = flags.MOVED_FROM | flags.MOVED_TO def __init__(self, path, options): super(INotifyWait, self).__init__() if str(path) == path: self.paths = [os.path.abspath(path)] else: self.paths = [] for p in path: self.path.append(os.path.abspath(p)) self.__active = True self.__process = None self.__inotify = INotify() # TODO: find a way to screen events emitted from unrelated paths for path in self.paths: self.__inotify.add_watch(path, options.get('events')) def on(self, type, callback): if self.__active: super().on(type, callback) if self.__process is not None: self.__process.terminate() self.__process = Process(target=self.__inotifyRead) self.__process.start() def stop(self): if self.__active: self.__active = False self.__process.terminate() self.__inotify.close() def __inotifyRead(self): while self.__active: for event in self.__inotify.read(): for flag in flags.from_mask(event.mask): for path in self.paths: self.emit(flag, { 'type': flag, 'path': path, 'entry': event.name })
def __init__(self, config, event_queue): super(FileMonitor, self).__init__() self.stop_request = threading.Event() self.inotify = INotify() self.event_queue = event_queue self.watch_paths = config.watch_paths self.watch_flags = config.watch_flags self.recursion = config.recursion self.watch_descriptor_paths = {}
def __init__(self, filepath): self.__path, self.__name = filepath.rsplit("/", 1) self.__inotify = INotify() watch_flags = \ Flags.MODIFY \ | Flags.CLOSE_WRITE \ | Flags.MOVED_TO \ | Flags.EXCL_UNLINK self.__wd = self.__inotify.add_watch(self.__path, watch_flags)
class TaskRegistrator(Queue): def __init__(self, pool_name): self.set_queue_config() self.__set_config(pool_name) def __set_config(self, pool_name): self.data = None self.queue_pool_dir = self.main_path + pool_name self.inotify = INotify() # watch_flags = flags.CREATE watch_flags = flags.CREATE | flags.MODIFY self.inotify.add_watch(self.queue_pool_dir, watch_flags) os.chdir(self.queue_pool_dir) def __extract_json(self, data): arr_data_raw = data.split(".") print(" ** arr_data_raw = ", arr_data_raw) arr_data = arr_data_raw[0].split("-") print(" ** arr_data = ", arr_data) return arr_data[0], arr_data[1] def run(self, pool_name): event = self.inotify.read()[0] file_path = "" print(" >>>>> event = ", event) for flag in flags.from_mask(event.mask): if str(flag) == "flags.CREATE" or str(flag) == "flags.MODIFY": try: Printer().cprint( " **** TaskRegistrator detected a new task to be registered ... name = %s" % event.name) cmd, key = self.__extract_json(event.name) file_path = self.main_path + pool_name + "/" + event.name print(" ** DIBACA = ", file_path) with open(file_path) as json_file: self.data = { "cmd": cmd, "key": key, "pool_name": pool_name, "path": file_path, "value": json.load(json_file) } print("Data .. DISIMPAN") return True except: print(" damn .. masuk else ...") pass if os.path.exists(file_path): print(" sampah! deleting ..") os.remove(file_path) print(" File is deleted ..") return True # return False def get_data(self): return self.data
def __init__(self, watch_dir, n_files): super().__init__(watch_dir, n_files) self.wd_to_path = {} self.inotify = INotify() watch_flags = flags.CREATE | flags.OPEN | flags.MODIFY wd = self.inotify.add_watch(self.watch_dir, watch_flags) self.wd_to_path[wd] = self.watch_dir
def run(self): LOGGER.info("starting") init_signals() ih = INotify() wds = {} register_watches(ih, wds) force = True events = [] while RUN: if force: force = False else: events = ih.read(1000) if events is None or len(events) == 0: create_idle_file() continue delete_idle_file() if len(events) > 0: LOGGER.info("got events") for event in events: try: path = wds[event.wd] except KeyError: path = "unknown" LOGGER.debug("%s on %s" % (event, path)) if not (event.mask & flags.IGNORED): if event.mask & flags.DELETE_SELF: try: path = wds[event.wd] unregister_watch(ih, wds, event.wd) register_watch(ih, wds, path) except KeyError: pass try: lock = filelock.FileLock(get_plugin_lock_path(), timeout=300) with lock.acquire(poll_intervall=1): # ok, there is no plugins.install/uninstall running pass except filelock.Timeout: LOGGER.warning("can't acquire plugin management lock => " "maybe a blocked plugins.install/uninstall ? " "=> exiting") break if not is_status_running_or_error(): LOGGER.info("The module is not RUNNING or ERROR => " "ignoring...") time.sleep(2) force = True continue ret = self.handle_event() if not ret: break register_watches(ih, wds) LOGGER.info("waiting for events...") LOGGER.info("stopped")
def __init__(self, timeout=None): # this could potentially be an LRU Cache self._store = OrderedDict() self._inotify = INotify() self._monitor = MonitorThread() self._monitor_thread = Thread(target=self._monitor.run, args=( self._inotify, self._store, timeout, ))
def __init__(self, root_dir=None, excluded_dirs=[]): if not root_dir: # TODO: create proper exception class raise NotADirectoryError self.root_dir = Directory(root_dir) self.excluded_dirs = excluded_dirs self.inotify = INotify() self.watch_flags = flags.CREATE | flags.MODIFY | flags.DELETE self.watched_dirs = {} self.wd_tracker = {} self.watch(self.root_dir)
def main(argv: List[str]) -> None: inotify = INotify() watch_flags = masks.ALL_EVENTS print("watching /tmp") inotify.add_watch('/tmp', watch_flags) while True: for event in inotify.read(): print(event) evs = [str(fl) for fl in flags.from_mask(event.mask)] print(" " + ", ".join(evs))
def __set_config(self, symbol, binary_id, id): self.binary_id = binary_id self.id = id self.tick_dir = local_settings["inotify_config"]["default_dir"] + symbol self.tick_file_path = self.tick_dir + "/" + local_settings[ "inotify_config"]["default_name"] self.inotify = INotify() watch_flags = flags.CREATE | flags.DELETE | flags.MODIFY | flags.DELETE_SELF self.inotify.add_watch(self.tick_dir, watch_flags) os.chdir(self.tick_dir) self.tick_data = None self.symbol = symbol
class Monitor(threading.Thread): def __init__(self, path_to_runtime_lib, register): threading.Thread.__init__(self) self.path = path_to_runtime_lib self.lastrun = 0 # get rid of this ugly hack self.register = register self.known_modules = set() def start_inotify(self): self.inotify = INotify() watch_flags = flags.CLOSE_WRITE | flags.DELETE self.wd = self.inotify.add_watch(self.path + '/', watch_flags) def run(self): self.alive = True try: while self.alive == True: for event in self.inotify.read(): if self.alive == False: break if event.name.endswith('.py'): print( 'Cheguei em event.name. Name: {}, knownmodules:{}'. format(event.name, self.known_modules)) module_name = self.path + '.' + os.path.splitext( event.name)[0] for flag in flags.from_mask(event.mask): if flag.name == 'CLOSE_WRITE': if event.name not in self.known_modules: print('Primeira vez em known modules ({})'. format(self.known_modules)) self.known_modules.add((event.name)) self.register.add_module(module_name) if flag.name == 'DELETE': # TODO: handle unload from roles / modules pass finally: self.inotify.rm_watch(self.wd) def stop(self): self.alive = False self.add_mock_file() self.remove_mock_file() # send a mock event to inotify in order to avoid the need to kill the thread self.join() def add_mock_file(self): with open('{}/___mocking_file.py'.format(self.path), 'w') as f: f.write("Mock") def remove_mock_file(self): os.remove('{}/___mocking_file.py'.format(self.path))
class McDirWatcher: def __init__(self): self.inotify = INotify() watch_flags = flags.CREATE | flags.MODIFY self.inotify.add_watch(LOG_DIR, watch_flags) def events(self): while True: yield from self.inotify.read() def close(self): self.inotify.close()
class FileMonitor(threading.Thread): def __init__(self, config, event_queue): super(FileMonitor, self).__init__() self.stop_request = threading.Event() self.inotify = INotify() self.event_queue = event_queue self.watch_paths = config.watch_paths self.watch_flags = config.watch_flags self.recursion = config.recursion self.watch_descriptor_paths = {} def run(self): for path in self.watch_paths: self.__add_watch(path) if self.recursion: for root, dirs, files in os.walk(path): if root not in self.watch_descriptor_paths: self.__add_watch(root) while not self.stop_request.isSet(): for event in self.inotify.read(timeout=5): file_event = FileEvent(event, self.watch_descriptor_paths[event[0]]) if self.recursion and flags.ISDIR & file_event.mask and flags.CREATE & file_event.mask: self.__add_watch(file_event.full_path) if flags.DELETE_SELF & file_event.mask: self.__rm_watch(file_event.full_path) self.event_queue.put(file_event) def __add_watch(self, path): wd = self.inotify.add_watch(path, self.watch_flags) self.watch_descriptor_paths[wd] = path self.watch_descriptor_paths[path] = wd for root, dirs, files in os.walk(path): if root not in self.watch_descriptor_paths: self.__add_watch(root) def __rm_watch(self, path): try: wd = self.watch_descriptor_paths[path] del self.watch_descriptor_paths[path] del self.watch_descriptor_paths[wd] except KeyError: if not path.endswith('/'): path = path + "/" wd = self.watch_descriptor_paths[path] del self.watch_descriptor_paths[path] del self.watch_descriptor_paths[wd] def join(self, timeout=None): self.stop_request.set() super(FileMonitor, self).join(timeout)
class INotifyQt(QObject): sig_event = pyqtSignal(inotify_simple.Event) def __init__(self): super().__init__() self.inotify = INotify() self.qnotifier = QSocketNotifier(self.inotify.fd, QSocketNotifier.Read) self.qnotifier.activated.connect(self._on_activated) self.wd = None def add_watch(self, path, flags=inotify_masks.ALL_EVENTS): self.wd = self.inotify.add_watch(path, flags) def _on_activated(self, fd): assert fd == self.inotify.fd for ev in self.inotify.read(): self.sig_event.emit(ev) def close(self): del self.qnotifier self.inotify.close()
def __init__(self, dir_filter, observer): self.inotify = INotify() self.watch_flags = flags.CREATE | flags.DELETE | flags.MODIFY | flags.DELETE_SELF self.watch_flags = masks.ALL_EVENTS self.watch_flags = \ flags.CREATE | \ flags.DELETE | \ flags.DELETE_SELF | \ flags.CLOSE_WRITE | \ flags.MOVE_SELF | \ flags.MOVED_FROM | \ flags.MOVED_TO | \ flags.EXCL_UNLINK self.wds = {} self.dir_filter = dir_filter self.observer = observer
def run(self): while not os.path.exists(self.watch_dir): logger.info("Waiting for %s...", self.watch_dir) time.sleep(30) inotify = INotify() watch_flags = flags.CLOSE_WRITE logger.info("Watching %s", self.watch_dir) try: inotify.add_watch(self.watch_dir, watch_flags) # And see the corresponding events: while True: events = inotify.read() matching = [f.name for f in events if f.name.startswith(self.prefix)] if len(matching) > 0: # take only the last file self.processForMovement(os.path.join(self.watch_dir, matching[-1])) inotify.close() except: logger.exception("Error when setting up inotify")
import os from inotify_simple import INotify, flags os.mkdir('/tmp/inotify_test') inotify = INotify() watch_flags = flags.CREATE | flags.DELETE | flags.MODIFY | flags.DELETE_SELF wd = inotify.add_watch('/tmp/inotify_test', watch_flags) # Now create, delete and modify some files in the directory being monitored: os.chdir('/tmp/inotify_test') # CREATE event for a directory: os.system('mkdir foo') # CREATE event for a file: os.system('echo hello > test.txt') # MODIFY event for the file: os.system('echo world >> test.txt') # DELETE event for the file os.system('rm test.txt') # DELETE event for the directory os.system('rmdir foo') os.chdir('/tmp') # DELETE_SELF on the original directory. # Also generates an IGNORED event # indicating the watch was removed. os.system('rmdir inotify_test') # And see the corresponding events: for event in inotify.read(): print(event) for flag in flags.from_mask(event.mask): print(' ' + str(flag))
def processForMovement(filename): #print(filename, file=sys.stderr) has_movement = processFile(filename, False) eg.reportMovement(has_movement) size = (82, 46) md = MotionDetector(100000, 30) eg = EventGen(30) watch_dir = '/run/replay/fragments' prefix = 'mv' if len(sys.argv) == 1: inotify = INotify() watch_flags = flags.CLOSE_WRITE wd = inotify.add_watch(watch_dir, watch_flags) # And see the corresponding events: while True: events = inotify.read() matching = [f.name for f in events if f.name.startswith(prefix)] if len(matching) > 0: # take only the last file processForMovement(os.path.join(watch_dir, matching[-1])) inotify.close() else: for f in sys.argv[1:]: processFile(f, True)
class Server(object): """A TCP server which manages, power, partitioning and scheduling of jobs on SpiNNaker machines. Once constructed the server starts a background thread (:py:attr:`._server_thread`, :py:meth:`._run`) which implements the main server logic and handles communication with clients, monitoring of asynchronous board control events (e.g. board power-on completion) and watches the config file for changes. All members of this object are assumed to be accessed only from this thread while it is running. The thread is stopped, and its completion awaited by calling :py:meth:`.stop_and_join`, stopping the server. The server uses a :py:class:`~spalloc_server.Controller` object to implement scheduling, allocation and machine management functionality. This object is :py:mod:`pickled <pickle>` when the server shuts down in order to preserve the state of all managed machines (e.g. allocated jobs etc.). To allow the interruption of the server thread on asynchronous events from the Controller a :py:func:`~socket.socketpair` (:py:attr:`._notify_send` and :py:attr:`._notify_send`) is used which monitored allong with client connections and config file changes. A number of callable commands are implemented by the server in the form of a subset of the :py:class:`.Server`'s methods indicated by the :py:func:`._command` decorator. These may be called by a client by sending a line ``{"command": "...", "args": [...], "kwargs": {...}}``. If the function throws an exception, the client is disconnected. If the function returns, it is packed as a JSON line ``{"return": ...}``. """ def __init__(self, config_filename, cold_start=False): """ Parameters ---------- config_filename : str The filename of the config file for the server which describes the machines to be controlled. cold_start : bool If False (the default), the server will attempt to restore its previous state, if True, the server will start from scratch. """ self._config_filename = config_filename self._cold_start = cold_start # Should the background thread terminate? self._stop = False # The background thread in which the server will run self._server_thread = threading.Thread(target=self._run, name="Server Thread") # The poll object used for listening for connections self._poll = select.poll() # This socket pair is used by background threads to interrupt the main # event loop. self._notify_send, self._notify_recv = socket.socketpair() self._poll.register(self._notify_recv, select.POLLIN) # Currently open sockets to clients. Once server started, should only # be accessed from the server thread. self._server_socket = None # {fd: socket, ...} self._client_sockets = {} # Buffered data received from each socket # {fd: buf, ...} self._client_buffers = {} # For each client, contains a set() of job IDs and machine names that # the client is watching for changes or None if all changes are to be # monitored. # {socket: set or None, ...} self._client_job_watches = {} self._client_machine_watches = {} # The current server configuration options. Once server started, should # only be accessed from the server thread. self._configuration = Configuration() # Infer the saved-state location self._state_filename = os.path.join( os.path.dirname(self._config_filename), ".{}.state.{}".format(os.path.basename(self._config_filename), __version__) ) # Attempt to restore saved state if required self._controller = None if not self._cold_start: if os.path.isfile(self._state_filename): try: with open(self._state_filename, "rb") as f: self._controller = pickle.load(f) logging.info("Server warm-starting from %s.", self._state_filename) except: # Some other error occurred during unpickling. logging.exception( "Server state could not be unpacked from %s.", self._state_filename) # Perform cold-start if no saved state was loaded if self._controller is None: logging.info("Server cold-starting.") self._controller = Controller() # Notify the background thread when something changes in the background # of the controller (e.g. power state changes). self._controller.on_background_state_change = self._notify # Read configuration file. This must succeed when the server is first # being started. if not self._read_config_file(): raise Exception("Config file could not be loaded.") # Set up inotify watcher for config file changes self._config_inotify = INotify() self._poll.register(self._config_inotify.fd, select.POLLIN) self._watch_config_file() # Start the server self._server_thread.start() # Flag for checking if the server is still alive self._running = True def _notify(self): """Notify the background thread that something has happened. Calling this method simply wakes up the server thread causing it to perform all its usual checks and processing steps. """ self._notify_send.send(b"x") def _watch_config_file(self): """Create an inotify watch on the config file. This watch is monitored by the main server threead and if the config file is changed, the config file is re-read. """ # A one-shot watch is used since some editors cause a delete event to # be produced when the file is saved, removing the watch anyway. Using # a one-shot watch simplifies implementation as it requires the watch # to *always* be recreated, rather than just 'sometimes'. self._config_inotify.add_watch(self._config_filename, inotify_flags.MODIFY | inotify_flags.ATTRIB | inotify_flags.CLOSE_WRITE | inotify_flags.MOVED_TO | inotify_flags.CREATE | inotify_flags.DELETE | inotify_flags.DELETE_SELF | inotify_flags.MOVE_SELF | inotify_flags.ONESHOT) def _read_config_file(self): """(Re-)read the server configuration. If reading of the config file fails, the current configuration is retained, unchanged. Returns ------- bool True if reading succeded, False otherwise. """ try: with open(self._config_filename, "r") as f: config_script = f.read() # pragma: no branch except (IOError, OSError): logging.exception("Could not read " "config file %s", self._config_filename) return False # The environment in which the configuration script is exexcuted (and # where the script will store its options.) try: g = {} g.update(configuration.__dict__) g.update(coordinates.__dict__) exec(config_script, g) except: # Executing the config file failed, don't update any settings logging.exception("Error while evaluating " "config file %s", self._config_filename) return False # Make sure a configuration object is specified new = g.get("configuration", None) if not isinstance(new, Configuration): logging.error("'configuration' must be a Configuration object " "in config file %s", self._config_filename) return False # Update the configuration old = self._configuration self._configuration = new # Restart the server if the port or IP has changed (or if the server # has not yet been started...) if (new.port != old.port or new.ip != old.ip or self._server_socket is None): # Close all open connections self._close() # Create a new server socket self._server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self._server_socket.bind((new.ip, new.port)) self._server_socket.listen(5) self._poll.register(self._server_socket, select.POLLIN) # Update the controller self._controller.max_retired_jobs = new.max_retired_jobs self._controller.machines = OrderedDict((m.name, m) for m in new.machines) logging.info("Config file %s read successfully.", self._config_filename) return True def _close(self): """Close all server sockets and disconnect all client connections.""" if self._server_socket is not None: self._poll.unregister(self._server_socket) self._server_socket.close() for client_socket in list(itervalues(self._client_sockets)): self._disconnect_client(client_socket) def _disconnect_client(self, client): """Disconnect a client. Parameters ---------- client : :py:class:`socket.Socket` """ try: logging.info("Client %s disconnected.", client.getpeername()) except: logging.info("Client %s disconnected.", client) # Remove from the client list del self._client_sockets[client.fileno()] # Clear input buffer del self._client_buffers[client] # Clear any watches self._client_job_watches.pop(client, None) self._client_machine_watches.pop(client, None) # Stop watching the client's socket for data self._poll.unregister(client) # Disconnect the client client.close() def _accept_client(self): """Accept a new client.""" client, addr = self._server_socket.accept() logging.info("New client connected from %s", addr) # Watch the client's socket for datat self._poll.register(client, select.POLLIN) # Keep a reference to the socket self._client_sockets[client.fileno()] = client # Create a buffer for data sent by the client self._client_buffers[client] = b"" def _handle_commands(self, client): """Handle an incomming command from a client. Parameters ---------- client : :py:class:`socket.Socket` """ try: data = client.recv(1024) except (OSError, IOError): data = b"" # Did the client disconnect? if len(data) == 0: self._disconnect_client(client) return self._client_buffers[client] += data # Process any complete commands (whole lines) while b"\n" in self._client_buffers[client]: line, _, self._client_buffers[client] = \ self._client_buffers[client].partition(b"\n") try: cmd_obj = json.loads(line.decode("utf-8")) # Execute the specified command ret_val = _COMMANDS[cmd_obj["command"]]( self, client, *cmd_obj["args"], **cmd_obj["kwargs"]) # Return the response client.send(json.dumps({"return": ret_val}).encode("utf-8") + b"\n") except: # If any of the above fails for any reason (e.g. invalid JSON, # unrecognised command, command crashes, etc.), just disconnect # the client. logging.exception("Client %s sent bad command %r, " "disconnecting", client.getpeername(), line) self._disconnect_client(client) return def _send_change_notifications(self): """Send any registered change notifications to clients. Sends notifications of the forms ``{"jobs_changed": [job_id, ...]}`` and ``{"machines_changed": [machine_name, ...]}`` to clients who have subscribed to be notified of changes to jobs or machines. """ # Notify clients about jobs which have changed changed_jobs = self._controller.changed_jobs if changed_jobs: for client, jobs in list(iteritems(self._client_job_watches)): if jobs is None or not jobs.isdisjoint(changed_jobs): try: client.send( json.dumps( {"jobs_changed": list(changed_jobs) if jobs is None else list(changed_jobs.intersection(jobs))} ).encode("utf-8") + b"\n") except (OSError, IOError): logging.exception( "Could not send notification.") self._disconnect_client(client) # Notify clients about machines which have changed changed_machines = self._controller.changed_machines if changed_machines: for client, machines in list( iteritems(self._client_machine_watches)): if (machines is None or not machines.isdisjoint(changed_machines)): try: client.send( json.dumps( {"machines_changed": list(changed_machines) if machines is None else list(changed_machines.intersection(machines))} ).encode("utf-8") + b"\n") except (OSError, IOError): logging.exception( "Could not send notification.") self._disconnect_client(client) def _run(self): """The main server thread. This 'infinate' loop runs in a background thread and waits for and processes events such as the :py:meth:`._notify` method being called, the config file changing, clients sending commands or new clients connecting. It also periodically calls destroy_timed_out_jobs on the controller. """ logging.info("Server running.") while not self._stop: # Wait for a connection to get opened/closed, a command to arrive, # the config file to change or the timeout to ellapse. events = self._poll.poll( self._configuration.timeout_check_interval) # Cull any jobs which have timed out self._controller.destroy_timed_out_jobs() for fd, event in events: if fd == self._notify_recv.fileno(): # _notify was called self._notify_recv.recv(1024) elif fd == self._server_socket.fileno(): # New client connected self._accept_client() elif fd in self._client_sockets: # Incoming data from client self._handle_commands(self._client_sockets[fd]) elif fd == self._config_inotify.fd: # Config file changed, re-read it time.sleep(0.1) self._config_inotify.read() self._watch_config_file() self._read_config_file() else: # pragma: no cover # Should not get here... assert False # Send any job/machine change notifications out self._send_change_notifications() def is_alive(self): """Is the server running?""" return self._running def join(self): """Wait for the server to completely shut down.""" self._server_thread.join() self._controller.join() def stop_and_join(self): """Stop the server and wait for it to shut down completely.""" logging.info("Server shutting down, please wait...") # Shut down server thread self._stop = True self._notify() self._server_thread.join() # Stop watching config file self._config_inotify.close() # Close all connections logging.info("Closing connections...") self._close() # Shut down the controller and flush all BMP commands logging.info("Waiting for all queued BMP commands...") self._controller.stop() self._controller.join() # Dump controller state to file with open(self._state_filename, "wb") as f: pickle.dump(self._controller, f) logging.info("Server shut down.") self._running = False @_command def version(self, client): """ Returns ------- str The server's version number.""" return __version__ @_command def create_job(self, client, *args, **kwargs): """Create a new job (i.e. allocation of boards). This function should be called in one of the following styles:: # Any single (SpiNN-5) board job_id = create_job(owner="me") job_id = create_job(1, owner="me") # Board x=3, y=2, z=1 on the machine named "m" job_id = create_job(3, 2, 1, machine="m", owner="me") # Any machine with at least 4 boards job_id = create_job(4, owner="me") # Any 7-or-more board machine with an aspect ratio at least as # square as 1:2 job_id = create_job(7, min_ratio=0.5, owner="me") # Any 4x5 triad segment of a machine (may or may-not be a # torus/full machine) job_id = create_job(4, 5, owner="me") # Any torus-connected (full machine) 4x2 machine job_id = create_job(4, 2, require_torus=True, owner="me") The 'other parameters' enumerated below may be used to further restrict what machines the job may be allocated onto. Jobs for which no suitable machines are available are immediately destroyed (and the reason given). Once a job has been created, it must be 'kept alive' by a simple watchdog_ mechanism. Jobs may be kept alive by periodically calling the :py:meth:`.job_keepalive` command or by calling any other job-specific command. Jobs are culled if no keep alive message is received for ``keepalive`` seconds. If absolutely necessary, a job's keepalive value may be set to None, disabling the keepalive mechanism. .. _watchdog: https://en.wikipedia.org/wiki/Watchdog_timer Once a job has been allocated some boards, these boards will be automatically powered on and left unbooted ready for use. Parameters ---------- owner : str **Required.** The name of the owner of this job. keepalive : float or None *Optional.* The maximum number of seconds which may elapse between a query on this job before it is automatically destroyed. If None, no timeout is used. (Default: 60.0) Other Parameters ---------------- machine : str or None *Optional.* Specify the name of a machine which this job must be executed on. If None, the first suitable machine available will be used, according to the tags selected below. Must be None when tags are given. (Default: None) tags : [str, ...] or None *Optional.* The set of tags which any machine running this job must have. If None is supplied, only machines with the "default" tag will be used. If machine is given, this argument must be None. (Default: None) min_ratio : float The aspect ratio (h/w) which the allocated region must be 'at least as square as'. Set to 0.0 for any allowable shape, 1.0 to be exactly square. Ignored when allocating single boards or specific rectangles of triads. max_dead_boards : int or None The maximum number of broken or unreachable boards to allow in the allocated region. If None, any number of dead boards is permitted, as long as the board on the bottom-left corner is alive (Default: None). max_dead_links : int or None The maximum number of broken links allow in the allocated region. When require_torus is True this includes wrap-around links, otherwise peripheral links are not counted. If None, any number of broken links is allowed. (Default: None). require_torus : bool If True, only allocate blocks with torus connectivity. In general this will only succeed for requests to allocate an entire machine (when the machine is otherwise not in use!). Must be False when allocating boards. (Default: False) Returns ------- int The job ID given to the newly allocated job. """ if kwargs.get("tags", None) is not None: kwargs["tags"] = set(kwargs["tags"]) return self._controller.create_job(*args, **kwargs) @_command def job_keepalive(self, client, job_id): """Reset the keepalive timer for the specified job. Note all other job-specific commands implicitly do this. """ self._controller.job_keepalive(job_id) @_command def get_job_state(self, client, job_id): """Poll the state of a running job. Returns ------- {"state": state, "power": power "keepalive": keepalive, "reason": reason} Where: state : :py:class:`~spalloc_server.controller.JobState` The current state of the queried job. power : bool or None If job is in the ready or power states, indicates whether the boards are power{ed,ing} on (True), or power{ed,ing} off (False). In other states, this value is None. keepalive : float or None The Job's keepalive value: the number of seconds between queries about the job before it is automatically destroyed. None if no timeout is active (or when the job has been destroyed). reason : str or None If the job has been destroyed, this may be a string describing the reason the job was terminated. start_time : float or None For queued and allocated jobs, gives the Unix time (UTC) at which the job was created (or None otherwise). """ out = self._controller.get_job_state(job_id)._asdict() out["state"] = int(out["state"]) return out @_command def get_job_machine_info(self, client, job_id): """Get the list of Ethernet connections to the allocated machine. Returns ------- {"width": width, "height": height, \ "connections": connections, "machine_name": machine_name} Where: width, height : int or None The dimensions of the machine in chips, e.g. for booting. None if no boards are allocated to the job. connections : [[[x, y], hostname], ...] or None A list giving Ethernet-connected chip coordinates in the machine to hostname. None if no boards are allocated to the job. machine_name : str or None The name of the machine the job is allocated on. None if no boards are allocated to the job. boards : [[x, y, z], ...] or None All the boards allocated to the job or None if no boards allocated. """ width, height, connections, machine_name, boards = \ self._controller.get_job_machine_info(job_id) if connections is not None: connections = list(iteritems(connections)) if boards is not None: boards = list(boards) return {"width": width, "height": height, "connections": connections, "machine_name": machine_name, "boards": boards} @_command def power_on_job_boards(self, client, job_id): """Power on (or reset if already on) boards associated with a job. Once called, the job will enter the 'power' state until the power state change is complete, this may take some time. """ self._controller.power_on_job_boards(job_id) @_command def power_off_job_boards(self, client, job_id): """Power off boards associated with a job. Once called, the job will enter the 'power' state until the power state change is complete, this may take some time. """ self._controller.power_off_job_boards(job_id) @_command def destroy_job(self, client, job_id, reason=None): """Destroy a job. Call when the job is finished, or to terminate it early, this function releases any resources consumed by the job and removes it from any queues. Parameters ---------- reason : str or None *Optional.* A human-readable string describing the reason for the job's destruction. """ self._controller.destroy_job(job_id, reason) @_command def notify_job(self, client, job_id=None): r"""Register to be notified about changes to a specific job ID. Once registered, a client will be asynchronously be sent notifications form ``{"jobs_changed": [job_id, ...]}\n`` enumerating job IDs which have changed. Notifications are sent when a job changes state, for example when created, queued, powering on/off, powered on and destroyed. The specific nature of the change is not reflected in the notification. Parameters ---------- job_id : int or None A job ID to be notified of or None if all job state changes should be reported. See Also -------- no_notify_job : Stop being notified about a job. notify_machine : Register to be notified about changes to machines. """ if job_id is None: self._client_job_watches[client] = None else: if client not in self._client_job_watches: self._client_job_watches[client] = set([job_id]) elif self._client_job_watches[client] is not None: self._client_job_watches[client].add(job_id) else: # Client is already notified about all changes, do nothing! pass @_command def no_notify_job(self, client, job_id=None): """Stop being notified about a specific job ID. Once this command returns, no further notifications for the specified ID will be received. Parameters ---------- job_id : id or None A job ID to no longer be notified of or None to not be notified of any jobs. Note that if all job IDs were registered for notification, this command only has an effect if the specified job_id is None. See Also -------- notify_job : Register to be notified about changes to a specific job. """ if client not in self._client_job_watches: return if job_id is None: del self._client_job_watches[client] else: watches = self._client_job_watches[client] if watches is not None: watches.discard(job_id) if len(watches) == 0: del self._client_job_watches[client] @_command def notify_machine(self, client, machine_name=None): r"""Register to be notified about a specific machine name. Once registered, a client will be asynchronously be sent notifications of the form ``{"machines_changed": [machine_name, ...]}\n`` enumerating machine names which have changed. Notifications are sent when a machine changes state, for example when created, change, removed, allocated a job or an allocated job is destroyed. Parameters ---------- machine_name : machine or None A machine name to be notified of or None if all machine state changes should be reported. See Also -------- no_notify_machine : Stop being notified about a machine. notify_job : Register to be notified about changes to jobs. """ if machine_name is None: self._client_machine_watches[client] = None else: if client not in self._client_machine_watches: self._client_machine_watches[client] = set([machine_name]) elif self._client_machine_watches[client] is not None: self._client_machine_watches[client].add(machine_name) else: # Client is already notified about all changes, do nothing! pass @_command def no_notify_machine(self, client, machine_name=None): """Unregister to be notified about a specific machine name. Once this command returns, no further notifications for the specified ID will be received. Parameters ---------- machine_name : name or None A machine name to no longer be notified of or None to not be notified of any machines. Note that if all machines were registered for notification, this command only has an effect if the specified machine_name is None. See Also -------- notify_machine : Register to be notified about changes to a machine. """ if client not in self._client_machine_watches: return if machine_name is None: del self._client_machine_watches[client] else: watches = self._client_machine_watches[client] if watches is not None: watches.discard(machine_name) if len(watches) == 0: del self._client_machine_watches[client] @_command def list_jobs(self, client): """Enumerate all non-destroyed jobs. Returns ------- jobs : [{...}, ...] A list of allocated/queued jobs in order of creation from oldest (first) to newest (last). Each job is described by a dictionary with the following keys: "job_id" is the ID of the job. "owner" is the string giving the name of the Job's owner. "start_time" is the time the job was created (Unix time, UTC). "keepalive" is the maximum time allowed between queries for this job before it is automatically destroyed (or None if the job can remain allocated indefinitely). "state" is the current :py:class:`~spalloc_server.controller.JobState` of the job. "power" indicates whether the boards are powered on or not. If job is in the ready or power states, indicates whether the boards are power{ed,ing} on (True), or power{ed,ing} off (False). In other states, this value is None. "args" and "kwargs" are the arguments to the alloc function which specifies the type/size of allocation requested and the restrictions on dead boards, links and torus connectivity. "allocated_machine_name" is the name of the machine the job has been allocated to run on (or None if not allocated yet). "boards" is a list [(x, y, z), ...] of boards allocated to the job. """ out = [] for job in self._controller.list_jobs(): job = job._asdict() job["state"] = int(job["state"]) if job["boards"] is not None: job["boards"] = list(job["boards"]) if job["kwargs"].get("tags", None) is not None: job["kwargs"]["tags"] = list(job["kwargs"]["tags"]) out.append(job) return out @_command def list_machines(self, client): """Enumerates all machines known to the system. Returns ------- machines : [{...}, ...] The list of machines known to the system in order of priority from highest (first) to lowest (last). Each machine is described by a dictionary with the following keys: "name" is the name of the machine. "tags" is the list ['tag', ...] of tags the machine has. "width" and "height" are the dimensions of the machine in triads. "dead_boards" is a list([(x, y, z), ...]) giving the coordinates of known-dead boards. "dead_links" is a list([(x, y, z, link), ...]) giving the locations of known-dead links from the perspective of the sender. Links to dead boards may or may not be included in this list. """ out = [] for machine in self._controller.list_machines(): machine = machine._asdict() machine["tags"] = list(machine["tags"]) machine["dead_boards"] = list(machine["dead_boards"]) machine["dead_links"] = [(x, y, z, int(link)) for x, y, z, link in machine["dead_links"]] out.append(machine) return out @_command def get_board_position(self, client, machine_name, x, y, z): """Get the physical location of a specified board. Parameters ---------- machine_name : str The name of the machine containing the board. x, y, z : int The logical board location within the machine. Returns ------- (cabinet, frame, board) or None The physical location of the board at the specified location or None if the machine/board are not recognised. """ return self._controller.get_board_position(machine_name, x, y, z) @_command def get_board_at_position(self, client, machine_name, x, y, z): """Get the logical location of a board at the specified physical location. Parameters ---------- machine_name : str The name of the machine containing the board. cabinet, frame, board : int The physical board location within the machine. Returns ------- (x, y, z) or None The logical location of the board at the specified location or None if the machine/board are not recognised. """ return self._controller.get_board_at_position(machine_name, x, y, z) @_command def where_is(self, client, **kwargs): """Find out where a SpiNNaker board or chip is located, logically and physically. May be called in one of the following styles:: >>> # Query by logical board coordinate within a machine. >>> where_is(machine=..., x=..., y=..., z=...) >>> # Query by physical board location within a machine. >>> where_is(machine=..., cabinet=..., frame=..., board=...) >>> # Query by chip coordinate (as if the machine were booted as >>> # one large machine). >>> where_is(machine=..., chip_x=..., chip_y=...) >>> # Query by chip coordinate, within the boards allocated to a >>> # job. >>> where_is(job_id=..., chip_x=..., chip_y=...) Returns ------- {"machine": ..., "logical": ..., "physical": ..., "chip": ..., \ "board_chip": ..., "job_chip": ..., "job_id": ...} or None If a board exists at the supplied location, a dictionary giving the location of the board/chip, supplied in a number of alternative forms. If the supplied coordinates do not specify a specific chip, the chip coordinates given are those of the Ethernet connected chip on that board. If no board exists at the supplied position, None is returned instead. ``machine`` gives the name of the machine containing the board. ``logical`` the logical board coordinate, (x, y, z) within the machine. ``physical`` the physical board location, (cabinet, frame, board), within the machine. ``chip`` the coordinates of the chip, (x, y), if the whole machine were booted as a single machine. ``board_chip`` the coordinates of the chip, (x, y), within its board. ``job_id`` is the job ID of the job currently allocated to the board identified or None if the board is not allocated to a job. ``job_chip`` the coordinates of the chip, (x, y), within its job, if a job is allocated to the board or None otherwise. """ return self._controller.where_is(**kwargs)
def __init__(self, config_filename, cold_start=False): """ Parameters ---------- config_filename : str The filename of the config file for the server which describes the machines to be controlled. cold_start : bool If False (the default), the server will attempt to restore its previous state, if True, the server will start from scratch. """ self._config_filename = config_filename self._cold_start = cold_start # Should the background thread terminate? self._stop = False # The background thread in which the server will run self._server_thread = threading.Thread(target=self._run, name="Server Thread") # The poll object used for listening for connections self._poll = select.poll() # This socket pair is used by background threads to interrupt the main # event loop. self._notify_send, self._notify_recv = socket.socketpair() self._poll.register(self._notify_recv, select.POLLIN) # Currently open sockets to clients. Once server started, should only # be accessed from the server thread. self._server_socket = None # {fd: socket, ...} self._client_sockets = {} # Buffered data received from each socket # {fd: buf, ...} self._client_buffers = {} # For each client, contains a set() of job IDs and machine names that # the client is watching for changes or None if all changes are to be # monitored. # {socket: set or None, ...} self._client_job_watches = {} self._client_machine_watches = {} # The current server configuration options. Once server started, should # only be accessed from the server thread. self._configuration = Configuration() # Infer the saved-state location self._state_filename = os.path.join( os.path.dirname(self._config_filename), ".{}.state.{}".format(os.path.basename(self._config_filename), __version__) ) # Attempt to restore saved state if required self._controller = None if not self._cold_start: if os.path.isfile(self._state_filename): try: with open(self._state_filename, "rb") as f: self._controller = pickle.load(f) logging.info("Server warm-starting from %s.", self._state_filename) except: # Some other error occurred during unpickling. logging.exception( "Server state could not be unpacked from %s.", self._state_filename) # Perform cold-start if no saved state was loaded if self._controller is None: logging.info("Server cold-starting.") self._controller = Controller() # Notify the background thread when something changes in the background # of the controller (e.g. power state changes). self._controller.on_background_state_change = self._notify # Read configuration file. This must succeed when the server is first # being started. if not self._read_config_file(): raise Exception("Config file could not be loaded.") # Set up inotify watcher for config file changes self._config_inotify = INotify() self._poll.register(self._config_inotify.fd, select.POLLIN) self._watch_config_file() # Start the server self._server_thread.start() # Flag for checking if the server is still alive self._running = True
device_id=row[0] print (device_id) s3.download_file('littercam','device-'+str(mac)+'/parameters.txt', 'parameters.txt') subprocess.Popen(["./Litter_detect"]) myMQTTClient.publish("topic", "connected", 0) timestr = time.strftime("%Y-%m-%d-%H:%M:%S") #timestr = time.strftime("%Y-%m-%d-%H:%M:%S") #s3.upload_file("/home/pi/detections/litter.jpg", "littercam", "device-"+str(mac)+"/litter"+str(timestr)+".jpg") # inotify initialization inotify = INotify() inotify_live = INotify() watch_flags = flags.CREATE | flags.MODIFY wd = inotify.add_watch('detections', watch_flags) wd_live = inotify_live.add_watch('live', watch_flags) #time.sleep(3) # Delays for 5 seconds. You can also use a float value. while 1: timestr = time.strftime("%Y-%m-%d-%H:%M:%S") JSONPayload = '{"state":{"desired":{"live": \"'+timestr+'\" }}}' deviceShadowHandler.shadowUpdate(JSONPayload,customShadowCallback_Update, 5) events= inotify.read(5000,2000) events_live= inotify_live.read(5000,2000) print ("evnets "+ str(events)) print ("evnets_live "+ str(events_live))