def _rescan(self, force=False): if not self.purrer: return # if pounce is on, tell the Purrer to rescan directories if self._pounce or force: dps = self.purrer.rescan() if dps: filenames = [dp.filename for dp in dps] dprint(2, "new data products:", filenames) self.message("Pounced on " + ", ".join(filenames)) if self.new_entry_dialog.addDataProducts(dps): dprint(2, "showing dialog") self.new_entry_dialog.show() # else read stuff from pipe for pipe in self.purrpipes.values(): do_show = False for command, show, content in pipe.read(): if command == "title": self.new_entry_dialog.suggestTitle(content) elif command == "comment": self.new_entry_dialog.addComment(content) elif command == "pounce": self.new_entry_dialog.addDataProducts( self.purrer.makeDataProducts([(content, not show)], unbanish=True)) else: print("Unknown command received from Purr pipe: ", command) continue do_show = do_show or show if do_show: self.new_entry_dialog.show()
def _rescan(self, force=False): if not self.purrer: return # if pounce is on, tell the Purrer to rescan directories if self._pounce or force: dps = self.purrer.rescan() if dps: filenames = [dp.filename for dp in dps] dprint(2, "new data products:", filenames) self.message("Pounced on " + ", ".join(filenames)) if self.new_entry_dialog.addDataProducts(dps): dprint(2, "showing dialog") self.new_entry_dialog.show() # else read stuff from pipe for pipe in self.purrpipes.values(): do_show = False for command, show, content in pipe.read(): if command == "title": self.new_entry_dialog.suggestTitle(content) elif command == "comment": self.new_entry_dialog.addComment(content) elif command == "pounce": self.new_entry_dialog.addDataProducts(self.purrer.makeDataProducts( [(content, not show)], unbanish=True)) else: print("Unknown command received from Purr pipe: ", command) continue do_show = do_show or show if do_show: self.new_entry_dialog.show()
def rescan(self): """Checks files and directories on watchlist for updates, rescans them for new data products. If any are found, returns them. Skips those in directories whose watchingState is set to Purr.UNWATCHED. """ if not self.attached: return dprint(5, "starting rescan") newstuff = {}; # this accumulates names of new or changed files. Keys are paths, values are 'quiet' flag. # store timestamp of scan self.last_scan_timestamp = time.time() # go through watched files/directories, check for mtime changes for path, watcher in list(self.watchers.items()): # get list of new files from watcher newfiles = watcher.newFiles() # None indicates access error, so drop it from watcher set if newfiles is None: if watcher.survive_deletion: dprintf(5, "access error on %s, but will still be watched\n", watcher.path) else: dprintf(2, "access error on %s, will no longer be watched\n", watcher.path) del self.watchers[path] if not watcher.disappeared: self.emit(SIGNAL("disappearedFile"), path) watcher.disappeared = True continue dprintf(5, "%s: %d new file(s)\n", watcher.path, len(newfiles)) # if a file has its own watcher, and is independently reported by a directory watcher, skip the directory's # version and let the file's watcher report it. Reason for this is that the file watcher may have a more # up-to-date timestamp, so we trust it over the dir watcher. newfiles = [p for p in newfiles if p is path or p not in self.watchers] # skip files in self._unwatched_paths newfiles = [filename for filename in newfiles if self._watching_state.get(os.path.dirname(filename)) > Purr.UNWATCHED] # Now go through files and add them to the newstuff dict for newfile in newfiles: # if quiet flag is explicitly set on watcher, enforce it # if not pouncing on directory, also add quietly if watcher.quiet or self._watching_state.get(os.path.dirname(newfile)) < Purr.POUNCE: quiet = True # else add quietly if file is not in the quiet patterns else: quiet = matches_patterns(os.path.basename(newfile), self._quiet_patterns) # add file to list of new products. Since a file may be reported by multiple # watchers, make the quiet flag a logical AND of all the quiet flags (i.e. DP will be # marked as quiet only if all watchers report it as quiet). newstuff[newfile] = quiet and newstuff.get(newfile, True) dprintf(4, "%s: new data product, quiet=%d (watcher quiet: %s)\n", newfile, quiet, watcher.quiet) # add a watcher for this file to the temp_watchers list. this is used below # to detect renamed and deleted files self.temp_watchers[newfile] = Purrer.WatchedFile(newfile) # now, go through temp_watchers to see if any newly pounced-on files have disappeared for path, watcher in list(self.temp_watchers.items()): # get list of new files from watcher if watcher.newFiles() is None: dprintf(2, "access error on %s, marking as disappeared", watcher.path) del self.temp_watchers[path] self.emit(SIGNAL("disappearedFile"), path) # if we have new data products, send them to the main window return self.makeDataProducts(iter(newstuff.items()))
def _cancelEditing (self): dprint(2,"cancelling editor"); if not self._editing: return; item0,column0 = self._editing; dprint(2,"cancelling editor for",item0.text(1),column0); self.closePersistentEditor(*self._editing); self._editing = None; item0.setText(column0,self._editing_oldtext);
def addWatchedDirectory(self, dirname, watching=Purr.WATCHED, save_config=True): """Starts watching the specified directories for changes""" # see if we're alredy watching this exact set of directories -- do nothing if so dirname = Purr.canonizePath(dirname) # do nothing if already watching if dirname in self.watched_dirs: dprint(1, "addWatchDirectory(): already watching %s\n", dirname) # watching=None means do not change the watch-state if watching is None: return else: if watching is None: watching = Purr.WATCHED # make watcher object wdir = Purrer.WatchedDir(dirname, mtime=self.timestamp, watch_patterns=self._watch_patterns, ignore_patterns=self._ignore_patterns) # fileset=None indicates error reading directory, so ignore it if wdir.fileset is None: print("There was an error reading the directory %s, will stop watching it." % dirname) self.setWatchingState(dirname, Purr.REMOVED, save_config=True) return self.watchers[dirname] = wdir self.watched_dirs.append(dirname) dprintf(2, "watching directory %s, mtime %s, %d files\n", dirname, time.strftime("%x %X", time.localtime(wdir.mtime)), len(wdir.fileset)) # find files in this directory matching the watch_patterns, and watch them for changes watchset = set() for patt in self._watch_patterns: watchset.update(fnmatch.filter(wdir.fileset, patt)) for fname in watchset: quiet = matches_patterns(fname, self._quiet_patterns) fullname = Purr.canonizePath(os.path.join(dirname, fname)) if fullname not in self.watchers: wfile = Purrer.WatchedFile(fullname, quiet=quiet, mtime=self.timestamp) self.watchers[fullname] = wfile dprintf(3, "watching file %s, timestamp %s, quiet %d\n", fullname, time.strftime("%x %X", time.localtime(wfile.mtime)), quiet) # find subdirectories matching the subdir_patterns, and watch them for changes for fname in wdir.fileset: fullname = Purr.canonizePath(os.path.join(dirname, fname)) if os.path.isdir(fullname): for desc, dir_patts, canary_patts in self._subdir_patterns: if matches_patterns(fname, dir_patts): quiet = matches_patterns(fname, self._quiet_patterns) wdir = Purrer.WatchedSubdir(fullname, canary_patterns=canary_patts, quiet=quiet, mtime=self.timestamp) self.watchers[fullname] = wdir dprintf(3, "watching subdirectory %s/{%s}, timestamp %s, quiet %d\n", fullname, ",".join(canary_patts), time.strftime("%x %X", time.localtime(wdir.mtime)), quiet) break # set state and save config self.setWatchingState(dirname, watching, save_config=save_config)
def setWatchedFilePatterns(self, watch, ignore=[]): self._watch = watch self._ignore = ignore self._watch_patterns = set() for desc, patts in self._watch: self._watch_patterns.update(patts) dprint(1, "watching patterns", self._watch_patterns) self._ignore_patterns = set() for desc, patts in self._ignore: self._ignore_patterns.update(patts) dprint(1, "ignoring patterns", self._ignore_patterns) Config.set("watch-patterns", make_pattern_list(self._watch)) Config.set("ignore-patterns", make_pattern_list(self._ignore))
def _startOrStopEditing (self,item=None,column=None): if self._editing: if self._editing == (item,column): return; else: item0,column0 = self._editing; dprint(2,"closing editor for",item0.text(1),column0); self.closePersistentEditor(*self._editing); self._editing = None; if column0 == self.ColRename: item0.setText(self.ColRename,_sanitizeFilename(str(item0.text(self.ColRename)))); if item and column in [self.ColRename,self.ColComment]: self._editing = item,column; dprint(2,"opening editor for",item.text(1),column); self._editing_oldtext = item.text(column); self.setCurrentItem(item,column); self.openPersistentEditor(item,column); self.itemWidget(item,column).setFocus(Qt.OtherFocusReason);
def dropEvent (self,ev): """Process drop event.""" # use function above to accept event if it contains a file URL files = self._checkDragDropEvent(ev); if files: pos = ev.pos(); dropitem = self.itemAt(pos); dprint(1,"dropped on",pos.x(),pos.y(),dropitem and str(dropitem.text(1))); # if event originated with ourselves, reorder items if ev.source() is self: self.reorderItems(dropitem,*files); # else event is from someone else, accept the dropped files else: self._dropped_on = dropitem; self.emit(SIGNAL("droppedFiles"),*files); # if event originated with another DPTreeWidget, emit a draggedAwayFiles() signal on its behalf if isinstance(ev.source(),DPTreeWidget): ev.source().emit(SIGNAL("draggedAwayFiles"),*files);
def setLogEntries(self, entries, save=True, update_policies=True): """Sets list of log entries. If save=True, saves the log. If update_policies=True, also updates default policies based on these entries""" prev = None; # "previous" valid entry for "Prev" link uplink = os.path.join("..", Purr.RenderIndex.INDEX); # "Up" link self.entries = [] for entry in entries: if not entry.ignore: entry.setPrevUpNextLinks(prev=prev, up=uplink) prev = entry self.entries.append(entry) if save: self.save() if update_policies: # populate default policies and renames based on entry list self._default_dp_props = {} for entry in entries: self.updatePoliciesFromEntry(entry) dprint(4, "default policies:", self._default_dp_props)
def __init__(self, purrlog, watchdirs=None): QObject.__init__(self) # load and parse configuration # watched files watch = Config.get( "watch-patterns", "Images=*fits,*FITS,*jpg,*png;TDL configuration=.tdl.conf") self._watch = parse_pattern_list(watch) self._watch_patterns = set() for desc, patts in self._watch: self._watch_patterns.update(patts) dprint(1, "watching patterns", self._watch_patterns) # quietly watched files (dialog is not popped up) watch = Config.get("watch-patterns-quiet", "TDL configuration=.tdl.conf") self._quiet = parse_pattern_list(watch) self._quiet_patterns = set() for desc, patts in self._quiet: self._quiet_patterns.update(patts) dprint(1, "quietly watching patterns", self._quiet_patterns) # ignored files ignore = Config.get( "ignore-patterns", "Hidden files=.*;Purr logs=*purrlog;MeqTree logs=meqtree.log;Python files=*.py*;Backup files=*~,*.bck;Measurement sets=*.MS,*.ms;CASA tables=table.f*,table.dat,table.info,table.lock" ) self._ignore = parse_pattern_list(ignore) self._ignore_patterns = set() for desc, patts in self._ignore: self._ignore_patterns.update(patts) dprint(1, "ignoring patterns", self._ignore_patterns) # watched subdirectories subdirs = Config.get("watch-subdirs", "MEP tables=*mep/funklets,table.dat") _re_patt = re.compile("^(.*)=(.*)/(.*)$") self._subdir_patterns = [] for ss in subdirs.split(';'): match = _re_patt.match(ss) if match: desc = match.group(1) dir_patt = match.group(2).split(',') canary_patt = match.group(3).split(',') self._subdir_patterns.append((desc, dir_patt, canary_patt)) dprint(1, "watching subdirectories", self._subdir_patterns) # attach to directories self.attached = False # will be True when we successfully attach self.other_lock = None # will be not None if another PURR holds a lock on this directory self.lockfile_fd = None self.lockfile_fobj = None self._attach(purrlog, watchdirs)
def setLogEntries(self, entries, save=True, update_policies=True): """Sets list of log entries. If save=True, saves the log. If update_policies=True, also updates default policies based on these entries""" prev = None # "previous" valid entry for "Prev" link uplink = os.path.join("..", Purr.RenderIndex.INDEX) # "Up" link self.entries = [] for entry in entries: if not entry.ignore: entry.setPrevUpNextLinks(prev=prev, up=uplink) prev = entry self.entries.append(entry) if save: self.save() if update_policies: # populate default policies and renames based on entry list self._default_dp_props = {} for entry in entries: self.updatePoliciesFromEntry(entry) dprint(4, "default policies:", self._default_dp_props)
def attachPurrlog (self,purrlog,watchdirs=[]): """Attaches Purr to the given purrlog directory. Arguments are passed to Purrer object as is.""" # check purrer stack for a Purrer already watching this directory dprint(1,"attaching to purrlog",purrlog); for i,purrer in enumerate(self.purrer_stack): if os.path.samefile(purrer.logdir,purrlog): dprint(1,"Purrer object found on stack (#%d),reusing\n",i); # found? move to front of stack self.purrer_stack.pop(i); self.purrer_stack.insert(0,purrer); # update purrer with watched directories, in case they have changed for dd in (watchdirs or []): purrer.addWatchedDirectory(dd,watching=None); break; # no purrer found, make a new one else: dprint(1,"creating new Purrer object"); try: purrer = Purr.Purrer(purrlog,watchdirs); except Purr.Purrer.LockedError,err: # check that we could attach, display message if not QMessageBox.warning(self,"Catfight!","""<P><NOBR>It appears that another PURR process (%s)</NOBR> is already attached to <tt>%s</tt>, so we're not allowed to touch it. You should exit the other PURR process first.</P>"""%(err.args[0],os.path.abspath(purrlog)),QMessageBox.Ok,0); return False; except Purr.Purrer.LockFailError,err: QMessageBox.warning(self,"Failed to obtain lock","""<P><NOBR>PURR was unable to obtain a lock</NOBR> on directory <tt>%s</tt> (error was "%s"). The most likely cause is insufficient permissions.</P>"""%(os.path.abspath(purrlog),err.args[0]),QMessageBox.Ok,0); return False;
def reorderItems (self,beforeitem,*files): # make list of items to be moved moving_items = []; for ff in files: item = self.dpitems.get(ff,None); # if dropping item on top of itself, ignore the operation if item is beforeitem: return; if item: dum = QWidget(); for col in self.ColAction,self.ColRender: widget = self.itemWidget(item,self.ColRender); widget and widget.setParent(dum); moving_items.append(self.takeTopLevelItem(self.indexOfTopLevelItem(item))); dprint(1,"moving items",[str(item.text(1)) for item in moving_items]); dprint(1,"to before",beforeitem and str(beforeitem.text(1))); # move them to specified location if moving_items: index = self.indexOfTopLevelItem(beforeitem) if beforeitem else self.topLevelItemCount(); self.insertTopLevelItems(index,moving_items); for item in moving_items: for col in self.ColAction,self.ColRender: self._itemComboBox(item,col); self.emit(SIGNAL("updated"));
def __init__(self, purrlog, watchdirs=None): QObject.__init__(self) # load and parse configuration # watched files watch = Config.get("watch-patterns", "Images=*fits,*FITS,*jpg,*png;TDL configuration=.tdl.conf") self._watch = parse_pattern_list(watch) self._watch_patterns = set() for desc, patts in self._watch: self._watch_patterns.update(patts) dprint(1, "watching patterns", self._watch_patterns) # quietly watched files (dialog is not popped up) watch = Config.get("watch-patterns-quiet", "TDL configuration=.tdl.conf") self._quiet = parse_pattern_list(watch) self._quiet_patterns = set() for desc, patts in self._quiet: self._quiet_patterns.update(patts) dprint(1, "quietly watching patterns", self._quiet_patterns) # ignored files ignore = Config.get("ignore-patterns", "Hidden files=.*;Purr logs=*purrlog;MeqTree logs=meqtree.log;Python files=*.py*;Backup files=*~,*.bck;Measurement sets=*.MS,*.ms;CASA tables=table.f*,table.dat,table.info,table.lock") self._ignore = parse_pattern_list(ignore) self._ignore_patterns = set() for desc, patts in self._ignore: self._ignore_patterns.update(patts) dprint(1, "ignoring patterns", self._ignore_patterns) # watched subdirectories subdirs = Config.get("watch-subdirs", "MEP tables=*mep/funklets,table.dat") _re_patt = re.compile("^(.*)=(.*)/(.*)$") self._subdir_patterns = [] for ss in subdirs.split(';'): match = _re_patt.match(ss) if match: desc = match.group(1) dir_patt = match.group(2).split(',') canary_patt = match.group(3).split(',') self._subdir_patterns.append((desc, dir_patt, canary_patt)) dprint(1, "watching subdirectories", self._subdir_patterns) # attach to directories self.attached = False; # will be True when we successfully attach self.other_lock = None; # will be not None if another PURR holds a lock on this directory self.lockfile_fd = None self.lockfile_fobj = None self._attach(purrlog, watchdirs)
def renderIndex(self, relpath="", refresh=0, refresh_index=0): """Returns HTML index code for this entry. If 'relpath' is empty, renders complete index.html file. If 'relpath' is not empty, then index is being included into a top-level log, and relpath should be passed to all sub-renderers. In this case the entry may make use of its cached_include file, if that is valid. If 'refresh' is set to a timestamp, then any subproducts (thumbnails, HTML caches, etc.) older than the timestamp will need to be regenerated. If 'refresh_index' is set to a timestamp, then any index files older than the timestamp will need to be regenerated. If 'relpath' is empty and 'prev', 'next' and/or 'up' is set, then Prev/Next/Up links will be inserted """ # check if cache can be used refresh_index = max(refresh, refresh_index) dprintf( 2, "%s: rendering HTML index with relpath='%s', refresh=%s refresh_index=%s\n", self.pathname, relpath, time.strftime("%x %X", time.localtime(refresh)), time.strftime("%x %X", time.localtime(refresh_index))) if relpath and self.cached_include_valid: try: if os.path.getmtime(self.cached_include) >= refresh_index: dprintf(2, "using include cache %s\n", self.cached_include) return open(self.cached_include).read() else: dprintf(2, "include cache %s out of date, will regenerate\n", self.cached_include) self.cached_include_valid = False except: print( "Error reading cached include code from %s, will regenerate" % self.cached_include) if verbosity.get_verbose() > 0: dprint(1, "Error traceback follows:") traceback.print_exc() self.cached_include_valid = False # form up attributes for % operator attrs = dict(self.__dict__) attrs['timestr'] = time.strftime("%x %X", time.localtime(self.timestamp)) attrs['relpath'] = relpath html = "" # replace title and comments for ignored entries if self.ignore: attrs['title'] = "This is not a real log entry" attrs['comment'] = """This entry was saved by PURR because the user chose to ignore and/or banish some data products. PURR has stored this information here for its opwn internal and highly nefarious purposes. This entry is will not appear in the log.""" # replace < and > in title and comments attrs['title'] = attrs['title'].replace("<", "<").replace(">", ">") # write header if asked if not relpath: icon = Purr.RenderIndex.renderIcon(24, "..") html += """<HTML><BODY> <TITLE>%(title)s</TITLE>""" % attrs if self._prev_link or self._next_link or self._up_link: html += """<DIV ALIGN=right><P>%s %s %s</P></DIV>""" % ( (self._prev_link and "<A HREF=\"%s\"><<Previous</A>" % self._prev_link) or "", (self._up_link and "<A HREF=\"%s\">Up</A>" % self._up_link) or "", (self._next_link and "<A HREF=\"%s\">Next>></A>" % self._next_link) or "") html += ( "<H2>" + icon + """ <A CLASS="TITLE" TIMESTAMP=%(timestamp)d>%(title)s</A></H2>""" ) % attrs else: icon = Purr.RenderIndex.renderIcon(24) html += """ <HR WIDTH=100%%> <H2>""" + icon + """ %(title)s</H2>""" % attrs # write comments html += """ <DIV ALIGN=right><P><SMALL>Logged on %(timestr)s</SMALL></P></DIV>\n <A CLASS="COMMENTS">\n""" % attrs # add comments logmode = False for cmt in self.comment.split("\n"): cmt = cmt.replace("<", "<").replace(">", ">").replace( "<BR>", "<BR>") html += """ <P>%s</P>\n""" % cmt html += """ </A>\n""" # add data products if self.dps: have_real_dps = bool([dp for dp in self.dps if not dp.ignored]) if have_real_dps: html += """ <H3>Data products</H3> <TABLE BORDER=1 FRAME=box RULES=all CELLPADDING=5>\n""" for dp in self.dps: dpattrs = dict(dp.__dict__) dpattrs['comment'] = dpattrs['comment'].replace("<", "<"). \ replace(">", ">").replace('"', "''") # if generating complete index, write empty anchor for each DP if not relpath: if dp.ignored: html += """ <A CLASS="DP" SRC="%(sourcepath)s" POLICY="%(policy)s" COMMENT="%(comment)s"></A>\n""" % dpattrs # write normal anchor for normal products else: dpattrs['relpath'] = relpath dpattrs['basename'] = os.path.basename(dp.filename) html += """ <A CLASS="DP" FILENAME="%(filename)s" SRC="%(sourcepath)s" POLICY="%(policy)s" QUIET=%(quiet)d TIMESTAMP=%(timestamp).6f RENDER="%(render)s" COMMENT="%(comment)s"></A>\n""" % dpattrs # render a table row if not dp.ignored: renderer = Purr.Render.makeRenderer(dp.render, dp, refresh=refresh) html += Purr.Render.renderInTable(renderer, relpath) if have_real_dps: html += """ </TABLE>""" # write footer if not relpath: html += "</BODY></HTML>\n" else: # now, write to include cache, if being included open(self.cached_include, 'w').write(html) self.cached_include_valid = True return html
def renderIndex(self, relpath="", refresh=0, refresh_index=0): """Returns HTML index code for this entry. If 'relpath' is empty, renders complete index.html file. If 'relpath' is not empty, then index is being included into a top-level log, and relpath should be passed to all sub-renderers. In this case the entry may make use of its cached_include file, if that is valid. If 'refresh' is set to a timestamp, then any subproducts (thumbnails, HTML caches, etc.) older than the timestamp will need to be regenerated. If 'refresh_index' is set to a timestamp, then any index files older than the timestamp will need to be regenerated. If 'relpath' is empty and 'prev', 'next' and/or 'up' is set, then Prev/Next/Up links will be inserted """ # check if cache can be used refresh_index = max(refresh, refresh_index) dprintf(2, "%s: rendering HTML index with relpath='%s', refresh=%s refresh_index=%s\n", self.pathname, relpath, time.strftime("%x %X", time.localtime(refresh)), time.strftime("%x %X", time.localtime(refresh_index))) if relpath and self.cached_include_valid: try: if os.path.getmtime(self.cached_include) >= refresh_index: dprintf(2, "using include cache %s\n", self.cached_include) return open(self.cached_include).read() else: dprintf(2, "include cache %s out of date, will regenerate\n", self.cached_include) self.cached_include_valid = False except: print("Error reading cached include code from %s, will regenerate" % self.cached_include) if verbosity.get_verbose() > 0: dprint(1, "Error traceback follows:") traceback.print_exc() self.cached_include_valid = False # form up attributes for % operator attrs = dict(self.__dict__) attrs['timestr'] = time.strftime("%x %X", time.localtime(self.timestamp)) attrs['relpath'] = relpath html = "" # replace title and comments for ignored entries if self.ignore: attrs['title'] = "This is not a real log entry" attrs['comment'] = """This entry was saved by PURR because the user chose to ignore and/or banish some data products. PURR has stored this information here for its opwn internal and highly nefarious purposes. This entry is will not appear in the log.""" # replace < and > in title and comments attrs['title'] = attrs['title'].replace("<", "<").replace(">", ">") # write header if asked if not relpath: icon = Purr.RenderIndex.renderIcon(24, "..") html += """<HTML><BODY> <TITLE>%(title)s</TITLE>""" % attrs if self._prev_link or self._next_link or self._up_link: html += """<DIV ALIGN=right><P>%s %s %s</P></DIV>""" % ( (self._prev_link and "<A HREF=\"%s\"><<Previous</A>" % self._prev_link) or "", (self._up_link and "<A HREF=\"%s\">Up</A>" % self._up_link) or "", (self._next_link and "<A HREF=\"%s\">Next>></A>" % self._next_link) or "" ) html += ("<H2>" + icon + """ <A CLASS="TITLE" TIMESTAMP=%(timestamp)d>%(title)s</A></H2>""") % attrs else: icon = Purr.RenderIndex.renderIcon(24) html += """ <HR WIDTH=100%%> <H2>""" + icon + """ %(title)s</H2>""" % attrs # write comments html += """ <DIV ALIGN=right><P><SMALL>Logged on %(timestr)s</SMALL></P></DIV>\n <A CLASS="COMMENTS">\n""" % attrs # add comments logmode = False for cmt in self.comment.split("\n"): cmt = cmt.replace("<", "<").replace(">", ">").replace("<BR>", "<BR>") html += """ <P>%s</P>\n""" % cmt html += """ </A>\n""" # add data products if self.dps: have_real_dps = bool([dp for dp in self.dps if not dp.ignored]) if have_real_dps: html += """ <H3>Data products</H3> <TABLE BORDER=1 FRAME=box RULES=all CELLPADDING=5>\n""" for dp in self.dps: dpattrs = dict(dp.__dict__) dpattrs['comment'] = dpattrs['comment'].replace("<", "<"). \ replace(">", ">").replace('"', "''") # if generating complete index, write empty anchor for each DP if not relpath: if dp.ignored: html += """ <A CLASS="DP" SRC="%(sourcepath)s" POLICY="%(policy)s" COMMENT="%(comment)s"></A>\n""" % dpattrs # write normal anchor for normal products else: dpattrs['relpath'] = relpath dpattrs['basename'] = os.path.basename(dp.filename) html += """ <A CLASS="DP" FILENAME="%(filename)s" SRC="%(sourcepath)s" POLICY="%(policy)s" QUIET=%(quiet)d TIMESTAMP=%(timestamp).6f RENDER="%(render)s" COMMENT="%(comment)s"></A>\n""" % dpattrs # render a table row if not dp.ignored: renderer = Purr.Render.makeRenderer(dp.render, dp, refresh=refresh) html += Purr.Render.renderInTable(renderer, relpath) if have_real_dps: html += """ </TABLE>""" # write footer if not relpath: html += "</BODY></HTML>\n" else: # now, write to include cache, if being included open(self.cached_include, 'w').write(html) self.cached_include_valid = True return html
def addWatchedDirectory(self, dirname, watching=Purr.WATCHED, save_config=True): """Starts watching the specified directories for changes""" # see if we're alredy watching this exact set of directories -- do nothing if so dirname = Purr.canonizePath(dirname) # do nothing if already watching if dirname in self.watched_dirs: dprint(1, "addWatchDirectory(): already watching %s\n", dirname) # watching=None means do not change the watch-state if watching is None: return else: if watching is None: watching = Purr.WATCHED # make watcher object wdir = Purrer.WatchedDir(dirname, mtime=self.timestamp, watch_patterns=self._watch_patterns, ignore_patterns=self._ignore_patterns) # fileset=None indicates error reading directory, so ignore it if wdir.fileset is None: print( "There was an error reading the directory %s, will stop watching it." % dirname) self.setWatchingState(dirname, Purr.REMOVED, save_config=True) return self.watchers[dirname] = wdir self.watched_dirs.append(dirname) dprintf(2, "watching directory %s, mtime %s, %d files\n", dirname, time.strftime("%x %X", time.localtime(wdir.mtime)), len(wdir.fileset)) # find files in this directory matching the watch_patterns, and watch them for changes watchset = set() for patt in self._watch_patterns: watchset.update(fnmatch.filter(wdir.fileset, patt)) for fname in watchset: quiet = matches_patterns(fname, self._quiet_patterns) fullname = Purr.canonizePath(os.path.join(dirname, fname)) if fullname not in self.watchers: wfile = Purrer.WatchedFile(fullname, quiet=quiet, mtime=self.timestamp) self.watchers[fullname] = wfile dprintf( 3, "watching file %s, timestamp %s, quiet %d\n", fullname, time.strftime("%x %X", time.localtime(wfile.mtime)), quiet) # find subdirectories matching the subdir_patterns, and watch them for changes for fname in wdir.fileset: fullname = Purr.canonizePath(os.path.join(dirname, fname)) if os.path.isdir(fullname): for desc, dir_patts, canary_patts in self._subdir_patterns: if matches_patterns(fname, dir_patts): quiet = matches_patterns(fname, self._quiet_patterns) wdir = Purrer.WatchedSubdir( fullname, canary_patterns=canary_patts, quiet=quiet, mtime=self.timestamp) self.watchers[fullname] = wdir dprintf( 3, "watching subdirectory %s/{%s}, timestamp %s, quiet %d\n", fullname, ",".join(canary_patts), time.strftime("%x %X", time.localtime(wdir.mtime)), quiet) break # set state and save config self.setWatchingState(dirname, watching, save_config=save_config)
self.purrer_stack.insert(0,purrer); # discard end of stack self.purrer_stack = self.purrer_stack[:3]; # attach signals self.connect(purrer,SIGNAL("disappearedFile"), self.new_entry_dialog.dropDataProducts); self.connect(purrer,SIGNAL("disappearedFile"), self.view_entry_dialog.dropDataProducts); # have we changed the current purrer? Update our state then # reopen Purr pipes self.purrpipes = {}; for dd,state in purrer.watchedDirectories(): self.purrpipes[dd] = Purr.Pipe.open(dd); if purrer is not self.purrer: self.message("Attached to %s"%purrer.logdir,ms=10000); dprint(1,"current Purrer changed, updating state"); # set window title path = Kittens.utils.collapseuser(os.path.join(purrer.logdir,'')); self.setWindowTitle("PURR - %s"%path); # other init self.purrer = purrer; self.new_entry_dialog.hide(); self.new_entry_dialog.reset(); dirs = [ path for path,state in purrer.watchedDirectories() ]; self.new_entry_dialog.setDefaultDirs(*dirs); self.view_entry_dialog.setDefaultDirs(*dirs); self.view_entry_dialog.hide(); self.viewer_dialog.hide(); self._viewing_ientry = None; self._setEntries(self.purrer.getLogEntries()); # print self._index_paths;
def _attach(self, purrlog, watchdirs=None): """Attaches Purr to a purrlog directory, and loads content. Returns False if nothing new has been loaded (because directory is the same), or True otherwise.""" purrlog = os.path.abspath(purrlog) dprint(1, "attaching to purrlog", purrlog) self.logdir = purrlog self.indexfile = os.path.join(self.logdir, "index.html") self.logtitle = "Unnamed log" self.timestamp = self.last_scan_timestamp = time.time() self._initIndexDir() # reset internal state self.ignorelistfile = None self.autopounce = False self.watched_dirs = [] self.entries = [] self._default_dp_props = {} self.watchers = {} self.temp_watchers = {} self.attached = False self._watching_state = {} # check that we hold a lock on the directory self.lockfile = os.path.join(self.logdir, ".purrlock") # try to open lock file for r/w try: self.lockfile_fd = os.open(self.lockfile, os.O_RDWR | os.O_CREAT) except: raise Purrer.LockFailError( "failed to open lock file %s for writing" % self.lockfile) # try to acquire lock on the lock file try: fcntl.lockf(self.lockfile_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except: other_lock = os.fdopen(self.lockfile_fd, 'r').read() self.lockfile_fd = None raise Purrer.LockedError(other_lock) # got lock, write our ID to the lock file global _lockstring try: self.lockfile_fobj = os.fdopen(self.lockfile_fd, 'w') self.lockfile_fobj.write(_lockstring) self.lockfile_fobj.flush() os.fsync(self.lockfile_fd) except: raise # raise Purrer.LockFailError("cannot write to lock file %s"%self.lockfile) # load log state if log directory already exists if os.path.exists(self.logdir): _busy = Purr.BusyIndicator() if os.path.exists(self.indexfile): try: parser = Purr.Parsers.LogIndexParser() for line in open(self.indexfile): parser.feed(line) self.logtitle = parser.title or self.logtitle self.timestamp = parser.timestamp or self.timestamp dprintf( 2, "attached log '%s', timestamp %s\n", self.logtitle, time.strftime("%x %X", time.localtime(self.timestamp))) except: traceback.print_exc() print("Error parsing %s, reverting to defaults" % self.indexfile) # load log entries entries = [] for fname in os.listdir(self.logdir): pathname = os.path.join(self.logdir, fname) if Purr.LogEntry.isValidPathname(pathname): try: entry = Purr.LogEntry(load=pathname) dprint(2, "loaded log entry", pathname) except: print("Error loading entry %s, skipping" % fname) traceback.print_exc() continue entries.append(entry) else: dprint(2, fname, "is not a valid Purr entry") # sort log entires by timestamp entries.sort(lambda a, b: cmp(a.timestamp, b.timestamp)) self.setLogEntries(entries, save=False) # update own timestamp if entries: self.timestamp = max(self.timestamp, entries[-1].timestamp) # else logfile doesn't exist, create it else: self._initIndexDir() # load configuration if it exists # init config file self.dirconfig = configparser.RawConfigParser() self.dirconfigfile = os.path.join(self.logdir, "dirconfig") if os.path.exists(self.dirconfigfile): try: self.dirconfig.read(self.dirconfigfile) except: print("Error loading config file %s" % self.dirconfigfile) traceback.print_exc() # load directory configuration for dirname in self.dirconfig.sections(): try: watching = self.dirconfig.getint(dirname, "watching") except: watching = Purr.WATCHED dirname = os.path.expanduser(dirname) self.addWatchedDirectory(dirname, watching, save_config=False) # start watching the specified directories for name in (watchdirs or []): self.addWatchedDirectory(name, watching=None) # Finally, go through list of ignored files and mark their watchers accordingly. # The ignorelist is a list of lines of the form "timestamp filename", giving the timestamp when a # file was last "ignored" by the purrlog user. self.ignorelistfile = os.path.join(self.logdir, "ignorelist") if os.path.exists(self.ignorelistfile): # read lines from file, ignore exceptions ignores = {} try: for line in open(self.ignorelistfile).readlines(): timestamp, policy, filename = line.strip().split(" ", 2) # update dictiornary with latest timestamp ignores[filename] = int(timestamp), policy except: print("Error reading %s" % self.ignorelistfile) traceback.print_exc() # now scan all listed files, and make sure their watchers' mtime is no older than the given # last-ignore-timestamp. This ensures that we don't pounce on these files after restarting purr. for filename, (timestamp, policy) in ignores.items(): watcher = self.watchers.get(filename, None) if watcher: watcher.mtime = max(watcher.mtime, timestamp) # init complete self.attached = True return True
def attachPurrlog(self, purrlog, watchdirs=[]): """Attaches Purr to the given purrlog directory. Arguments are passed to Purrer object as is.""" # check purrer stack for a Purrer already watching this directory dprint(1, "attaching to purrlog", purrlog) for i, purrer in enumerate(self.purrer_stack): if os.path.samefile(purrer.logdir, purrlog): dprint(1, "Purrer object found on stack (#%d),reusing\n", i) # found? move to front of stack self.purrer_stack.pop(i) self.purrer_stack.insert(0, purrer) # update purrer with watched directories, in case they have changed for dd in (watchdirs or []): purrer.addWatchedDirectory(dd, watching=None) break # no purrer found, make a new one else: dprint(1, "creating new Purrer object") try: purrer = Purr.Purrer(purrlog, watchdirs) except Purr.Purrer.LockedError as err: # check that we could attach, display message if not QMessageBox.warning(self, "Catfight!", """<P><NOBR>It appears that another PURR process (%s)</NOBR> is already attached to <tt>%s</tt>, so we're not allowed to touch it. You should exit the other PURR process first.</P>""" % (err.args[0], os.path.abspath(purrlog)), QMessageBox.Ok, 0) return False except Purr.Purrer.LockFailError as err: QMessageBox.warning(self, "Failed to obtain lock", """<P><NOBR>PURR was unable to obtain a lock</NOBR> on directory <tt>%s</tt> (error was "%s"). The most likely cause is insufficient permissions.</P>""" % ( os.path.abspath(purrlog), err.args[0]), QMessageBox.Ok, 0) return False self.purrer_stack.insert(0, purrer) # discard end of stack self.purrer_stack = self.purrer_stack[:3] # attach signals self.connect(purrer, SIGNAL("disappearedFile"), self.new_entry_dialog.dropDataProducts) self.connect(purrer, SIGNAL("disappearedFile"), self.view_entry_dialog.dropDataProducts) # have we changed the current purrer? Update our state then # reopen Purr pipes self.purrpipes = {} for dd, state in purrer.watchedDirectories(): self.purrpipes[dd] = Purr.Pipe.open(dd) if purrer is not self.purrer: self.message("Attached to %s" % purrer.logdir, ms=10000) dprint(1, "current Purrer changed, updating state") # set window title path = Kittens.utils.collapseuser(os.path.join(purrer.logdir, '')) self.setWindowTitle("PURR - %s" % path) # other init self.purrer = purrer self.new_entry_dialog.hide() self.new_entry_dialog.reset() dirs = [path for path, state in purrer.watchedDirectories()] self.new_entry_dialog.setDefaultDirs(*dirs) self.view_entry_dialog.setDefaultDirs(*dirs) self.view_entry_dialog.hide() self.viewer_dialog.hide() self._viewing_ientry = None self._setEntries(self.purrer.getLogEntries()) # print self._index_paths self._viewer_timestamp = None self._updateViewer() self._updateNames() # update directory widgets self.wdirlist.clear() for pathname, state in purrer.watchedDirectories(): self.wdirlist.add(pathname, state) # Reset _pounce to false -- this will cause checkPounceStatus() into a rescan self._pounce = False self._checkPounceStatus() return True
def _initIndexDir(self): """makes sure purrlog directory is properly set up""" if not os.path.exists(self.logdir): os.mkdir(self.logdir) dprint(1, "created", self.logdir) Purr.RenderIndex.initIndexDir(self.logdir)
def _itemActivated (self,item,col): dprint(2,"_itemActivated",item.text(1),col); self._startOrStopEditing(item,col);
def _attach(self, purrlog, watchdirs=None): """Attaches Purr to a purrlog directory, and loads content. Returns False if nothing new has been loaded (because directory is the same), or True otherwise.""" purrlog = os.path.abspath(purrlog) dprint(1, "attaching to purrlog", purrlog) self.logdir = purrlog self.indexfile = os.path.join(self.logdir, "index.html") self.logtitle = "Unnamed log" self.timestamp = self.last_scan_timestamp = time.time() self._initIndexDir() # reset internal state self.ignorelistfile = None self.autopounce = False self.watched_dirs = [] self.entries = [] self._default_dp_props = {} self.watchers = {} self.temp_watchers = {} self.attached = False self._watching_state = {} # check that we hold a lock on the directory self.lockfile = os.path.join(self.logdir, ".purrlock") # try to open lock file for r/w try: self.lockfile_fd = os.open(self.lockfile, os.O_RDWR | os.O_CREAT) except: raise Purrer.LockFailError("failed to open lock file %s for writing" % self.lockfile) # try to acquire lock on the lock file try: fcntl.lockf(self.lockfile_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except: other_lock = os.fdopen(self.lockfile_fd, 'r').read() self.lockfile_fd = None raise Purrer.LockedError(other_lock) # got lock, write our ID to the lock file global _lockstring try: self.lockfile_fobj = os.fdopen(self.lockfile_fd, 'w') self.lockfile_fobj.write(_lockstring) self.lockfile_fobj.flush() os.fsync(self.lockfile_fd) except: raise # raise Purrer.LockFailError("cannot write to lock file %s"%self.lockfile) # load log state if log directory already exists if os.path.exists(self.logdir): _busy = Purr.BusyIndicator() if os.path.exists(self.indexfile): try: parser = Purr.Parsers.LogIndexParser() for line in open(self.indexfile): parser.feed(line) self.logtitle = parser.title or self.logtitle self.timestamp = parser.timestamp or self.timestamp dprintf(2, "attached log '%s', timestamp %s\n", self.logtitle, time.strftime("%x %X", time.localtime(self.timestamp))) except: traceback.print_exc() print("Error parsing %s, reverting to defaults" % self.indexfile) # load log entries entries = [] for fname in os.listdir(self.logdir): pathname = os.path.join(self.logdir, fname) if Purr.LogEntry.isValidPathname(pathname): try: entry = Purr.LogEntry(load=pathname) dprint(2, "loaded log entry", pathname) except: print("Error loading entry %s, skipping" % fname) traceback.print_exc() continue entries.append(entry) else: dprint(2, fname, "is not a valid Purr entry") # sort log entires by timestamp entries.sort(lambda a, b: cmp(a.timestamp, b.timestamp)) self.setLogEntries(entries, save=False) # update own timestamp if entries: self.timestamp = max(self.timestamp, entries[-1].timestamp) # else logfile doesn't exist, create it else: self._initIndexDir() # load configuration if it exists # init config file self.dirconfig = configparser.RawConfigParser() self.dirconfigfile = os.path.join(self.logdir, "dirconfig") if os.path.exists(self.dirconfigfile): try: self.dirconfig.read(self.dirconfigfile) except: print("Error loading config file %s" % self.dirconfigfile) traceback.print_exc() # load directory configuration for dirname in self.dirconfig.sections(): try: watching = self.dirconfig.getint(dirname, "watching") except: watching = Purr.WATCHED dirname = os.path.expanduser(dirname) self.addWatchedDirectory(dirname, watching, save_config=False) # start watching the specified directories for name in (watchdirs or []): self.addWatchedDirectory(name, watching=None) # Finally, go through list of ignored files and mark their watchers accordingly. # The ignorelist is a list of lines of the form "timestamp filename", giving the timestamp when a # file was last "ignored" by the purrlog user. self.ignorelistfile = os.path.join(self.logdir, "ignorelist") if os.path.exists(self.ignorelistfile): # read lines from file, ignore exceptions ignores = {} try: for line in open(self.ignorelistfile).readlines(): timestamp, policy, filename = line.strip().split(" ", 2) # update dictiornary with latest timestamp ignores[filename] = int(timestamp), policy except: print("Error reading %s" % self.ignorelistfile) traceback.print_exc() # now scan all listed files, and make sure their watchers' mtime is no older than the given # last-ignore-timestamp. This ensures that we don't pounce on these files after restarting purr. for filename, (timestamp, policy) in ignores.items(): watcher = self.watchers.get(filename, None) if watcher: watcher.mtime = max(watcher.mtime, timestamp) # init complete self.attached = True return True
def rescan(self): """Checks files and directories on watchlist for updates, rescans them for new data products. If any are found, returns them. Skips those in directories whose watchingState is set to Purr.UNWATCHED. """ if not self.attached: return dprint(5, "starting rescan") newstuff = {} # this accumulates names of new or changed files. Keys are paths, values are 'quiet' flag. # store timestamp of scan self.last_scan_timestamp = time.time() # go through watched files/directories, check for mtime changes for path, watcher in list(self.watchers.items()): # get list of new files from watcher newfiles = watcher.newFiles() # None indicates access error, so drop it from watcher set if newfiles is None: if watcher.survive_deletion: dprintf(5, "access error on %s, but will still be watched\n", watcher.path) else: dprintf(2, "access error on %s, will no longer be watched\n", watcher.path) del self.watchers[path] if not watcher.disappeared: self.emit(SIGNAL("disappearedFile"), path) watcher.disappeared = True continue dprintf(5, "%s: %d new file(s)\n", watcher.path, len(newfiles)) # if a file has its own watcher, and is independently reported by a directory watcher, skip the directory's # version and let the file's watcher report it. Reason for this is that the file watcher may have a more # up-to-date timestamp, so we trust it over the dir watcher. newfiles = [ p for p in newfiles if p is path or p not in self.watchers ] # skip files in self._unwatched_paths newfiles = [ filename for filename in newfiles if self._watching_state.get( os.path.dirname(filename)) > Purr.UNWATCHED ] # Now go through files and add them to the newstuff dict for newfile in newfiles: # if quiet flag is explicitly set on watcher, enforce it # if not pouncing on directory, also add quietly if watcher.quiet or self._watching_state.get( os.path.dirname(newfile)) < Purr.POUNCE: quiet = True # else add quietly if file is not in the quiet patterns else: quiet = matches_patterns(os.path.basename(newfile), self._quiet_patterns) # add file to list of new products. Since a file may be reported by multiple # watchers, make the quiet flag a logical AND of all the quiet flags (i.e. DP will be # marked as quiet only if all watchers report it as quiet). newstuff[newfile] = quiet and newstuff.get(newfile, True) dprintf( 4, "%s: new data product, quiet=%d (watcher quiet: %s)\n", newfile, quiet, watcher.quiet) # add a watcher for this file to the temp_watchers list. this is used below # to detect renamed and deleted files self.temp_watchers[newfile] = Purrer.WatchedFile(newfile) # now, go through temp_watchers to see if any newly pounced-on files have disappeared for path, watcher in list(self.temp_watchers.items()): # get list of new files from watcher if watcher.newFiles() is None: dprintf(2, "access error on %s, marking as disappeared", watcher.path) del self.temp_watchers[path] self.emit(SIGNAL("disappearedFile"), path) # if we have new data products, send them to the main window return self.makeDataProducts(iter(newstuff.items()))
def attachPurrlog(self, purrlog, watchdirs=[]): """Attaches Purr to the given purrlog directory. Arguments are passed to Purrer object as is.""" # check purrer stack for a Purrer already watching this directory dprint(1, "attaching to purrlog", purrlog) for i, purrer in enumerate(self.purrer_stack): if os.path.samefile(purrer.logdir, purrlog): dprint(1, "Purrer object found on stack (#%d),reusing\n", i) # found? move to front of stack self.purrer_stack.pop(i) self.purrer_stack.insert(0, purrer) # update purrer with watched directories, in case they have changed for dd in (watchdirs or []): purrer.addWatchedDirectory(dd, watching=None) break # no purrer found, make a new one else: dprint(1, "creating new Purrer object") try: purrer = Purr.Purrer(purrlog, watchdirs) except Purr.Purrer.LockedError as err: # check that we could attach, display message if not QMessageBox.warning( self, "Catfight!", """<P><NOBR>It appears that another PURR process (%s)</NOBR> is already attached to <tt>%s</tt>, so we're not allowed to touch it. You should exit the other PURR process first.</P>""" % (err.args[0], os.path.abspath(purrlog)), QMessageBox.Ok, 0) return False except Purr.Purrer.LockFailError as err: QMessageBox.warning( self, "Failed to obtain lock", """<P><NOBR>PURR was unable to obtain a lock</NOBR> on directory <tt>%s</tt> (error was "%s"). The most likely cause is insufficient permissions.</P>""" % (os.path.abspath(purrlog), err.args[0]), QMessageBox.Ok, 0) return False self.purrer_stack.insert(0, purrer) # discard end of stack self.purrer_stack = self.purrer_stack[:3] # attach signals self.connect(purrer, SIGNAL("disappearedFile"), self.new_entry_dialog.dropDataProducts) self.connect(purrer, SIGNAL("disappearedFile"), self.view_entry_dialog.dropDataProducts) # have we changed the current purrer? Update our state then # reopen Purr pipes self.purrpipes = {} for dd, state in purrer.watchedDirectories(): self.purrpipes[dd] = Purr.Pipe.open(dd) if purrer is not self.purrer: self.message("Attached to %s" % purrer.logdir, ms=10000) dprint(1, "current Purrer changed, updating state") # set window title path = Kittens.utils.collapseuser(os.path.join(purrer.logdir, '')) self.setWindowTitle("PURR - %s" % path) # other init self.purrer = purrer self.new_entry_dialog.hide() self.new_entry_dialog.reset() dirs = [path for path, state in purrer.watchedDirectories()] self.new_entry_dialog.setDefaultDirs(*dirs) self.view_entry_dialog.setDefaultDirs(*dirs) self.view_entry_dialog.hide() self.viewer_dialog.hide() self._viewing_ientry = None self._setEntries(self.purrer.getLogEntries()) # print self._index_paths self._viewer_timestamp = None self._updateViewer() self._updateNames() # update directory widgets self.wdirlist.clear() for pathname, state in purrer.watchedDirectories(): self.wdirlist.add(pathname, state) # Reset _pounce to false -- this will cause checkPounceStatus() into a rescan self._pounce = False self._checkPounceStatus() return True