def makeDataProducts(self, files, unbanish=False, unignore=False):
"""makes a list of DPs from a list of (filename,quiet) pairs.
If unbanish is False, DPs with a default "banish" policy will be skipped.
Symlinks will be resolved, and non-unique filenames removed from list.
"""
paths = set()
dps = []
for filename, quiet in files:
filename = filename.rstrip('/')
sourcepath = Purr.canonizePath(filename)
if sourcepath not in paths:
paths.add(sourcepath)
filename = os.path.basename(filename)
policy, filename, comment = self._default_dp_props.get(
filename, ("copy", filename, ""))
dprintf(4, "%s: default policy is %s,%s,%s\n", sourcepath,
policy, filename, comment)
if policy == "banish":
if unbanish:
policy = "copy"
else:
continue
if unignore and policy == "ignore":
policy = "copy"
dps.append(
Purr.DataProduct(filename=filename,
sourcepath=sourcepath,
policy=policy,
comment=comment,
quiet=quiet))
return sorted(dps, lambda a, b: cmp(a.filename, b.filename))
def makeDataProducts(self, files, unbanish=False, unignore=False):
"""makes a list of DPs from a list of (filename,quiet) pairs.
If unbanish is False, DPs with a default "banish" policy will be skipped.
Symlinks will be resolved, and non-unique filenames removed from list.
"""
paths = set()
dps = []
for filename, quiet in files:
filename = filename.rstrip('/')
sourcepath = Purr.canonizePath(filename)
if sourcepath not in paths:
paths.add(sourcepath)
filename = os.path.basename(filename)
policy, filename, comment = self._default_dp_props.get(filename, ("copy", filename, ""))
dprintf(4, "%s: default policy is %s,%s,%s\n", sourcepath, policy, filename, comment)
if policy == "banish":
if unbanish:
policy = "copy"
else:
continue
if unignore and policy == "ignore":
policy = "copy"
dps.append(Purr.DataProduct(filename=filename, sourcepath=sourcepath,
policy=policy, comment=comment, quiet=quiet))
return sorted(dps, lambda a, b: cmp(a.filename, b.filename))
def rescan(self):
"""Checks files and directories on watchlist for updates, rescans them for new data products.
If any are found, returns them. Skips those in directories whose watchingState is set to Purr.UNWATCHED.
"""
if not self.attached:
return
dprint(5, "starting rescan")
newstuff = {}; # this accumulates names of new or changed files. Keys are paths, values are 'quiet' flag.
# store timestamp of scan
self.last_scan_timestamp = time.time()
# go through watched files/directories, check for mtime changes
for path, watcher in list(self.watchers.items()):
# get list of new files from watcher
newfiles = watcher.newFiles()
# None indicates access error, so drop it from watcher set
if newfiles is None:
if watcher.survive_deletion:
dprintf(5, "access error on %s, but will still be watched\n", watcher.path)
else:
dprintf(2, "access error on %s, will no longer be watched\n", watcher.path)
del self.watchers[path]
if not watcher.disappeared:
self.emit(SIGNAL("disappearedFile"), path)
watcher.disappeared = True
continue
dprintf(5, "%s: %d new file(s)\n", watcher.path, len(newfiles))
# if a file has its own watcher, and is independently reported by a directory watcher, skip the directory's
# version and let the file's watcher report it. Reason for this is that the file watcher may have a more
# up-to-date timestamp, so we trust it over the dir watcher.
newfiles = [p for p in newfiles if p is path or p not in self.watchers]
# skip files in self._unwatched_paths
newfiles = [filename for filename in newfiles if
self._watching_state.get(os.path.dirname(filename)) > Purr.UNWATCHED]
# Now go through files and add them to the newstuff dict
for newfile in newfiles:
# if quiet flag is explicitly set on watcher, enforce it
# if not pouncing on directory, also add quietly
if watcher.quiet or self._watching_state.get(os.path.dirname(newfile)) < Purr.POUNCE:
quiet = True
# else add quietly if file is not in the quiet patterns
else:
quiet = matches_patterns(os.path.basename(newfile), self._quiet_patterns)
# add file to list of new products. Since a file may be reported by multiple
# watchers, make the quiet flag a logical AND of all the quiet flags (i.e. DP will be
# marked as quiet only if all watchers report it as quiet).
newstuff[newfile] = quiet and newstuff.get(newfile, True)
dprintf(4, "%s: new data product, quiet=%d (watcher quiet: %s)\n", newfile, quiet, watcher.quiet)
# add a watcher for this file to the temp_watchers list. this is used below
# to detect renamed and deleted files
self.temp_watchers[newfile] = Purrer.WatchedFile(newfile)
# now, go through temp_watchers to see if any newly pounced-on files have disappeared
for path, watcher in list(self.temp_watchers.items()):
# get list of new files from watcher
if watcher.newFiles() is None:
dprintf(2, "access error on %s, marking as disappeared", watcher.path)
del self.temp_watchers[path]
self.emit(SIGNAL("disappearedFile"), path)
# if we have new data products, send them to the main window
return self.makeDataProducts(iter(newstuff.items()))
def __init__(self, path, canary_patterns=[], **kw):
Purrer.WatchedDir.__init__(self, path, **kw)
self.canary_patterns = canary_patterns
self.canaries = {}
# if no read errors, make up list of canaries from canary patterns
if self.fileset is not None:
for fname in self.fileset:
if matches_patterns(fname, canary_patterns):
fullname = os.path.join(self.path, fname)
self.canaries[fullname] = Purrer.WatchedFile(fullname, mtime=self.mtime)
dprintf(3, "watching canary file %s, timestamp %s\n",
fullname, time.strftime("%x %X", time.localtime(self.mtime)))
def setWatchingState(self, path, watching, save_config=True):
dprintf(2, "%s: watching state is %d\n", path, watching)
self._watching_state[path] = watching
path = Kittens.utils.collapseuser(path)
if watching == Purr.REMOVED:
if self.dirconfig.has_section(path):
self.dirconfig.remove_section(path)
else:
if not self.dirconfig.has_section(path):
self.dirconfig.add_section(path)
self.dirconfig.set(path, "watching", str(watching))
if save_config:
self.dirconfig.write(file(self.dirconfigfile, 'wt'))
def setWatchingState(self, path, watching, save_config=True):
dprintf(2, "%s: watching state is %d\n", path, watching)
self._watching_state[path] = watching
path = Kittens.utils.collapseuser(path)
if watching == Purr.REMOVED:
if self.dirconfig.has_section(path):
self.dirconfig.remove_section(path)
else:
if not self.dirconfig.has_section(path):
self.dirconfig.add_section(path)
self.dirconfig.set(path, "watching", str(watching))
if save_config:
self.dirconfig.write(file(self.dirconfigfile, 'wt'))
def __init__(self, path, canary_patterns=[], **kw):
Purrer.WatchedDir.__init__(self, path, **kw)
self.canary_patterns = canary_patterns
self.canaries = {}
# if no read errors, make up list of canaries from canary patterns
if self.fileset is not None:
for fname in self.fileset:
if matches_patterns(fname, canary_patterns):
fullname = os.path.join(self.path, fname)
self.canaries[fullname] = Purrer.WatchedFile(
fullname, mtime=self.mtime)
dprintf(
3, "watching canary file %s, timestamp %s\n",
fullname,
time.strftime("%x %X", time.localtime(self.mtime)))
def __init__ (self,path,quiet=None,mtime=None,survive_deletion=False):
"""Creates watched file at 'path'. The 'quiet' flag is simply stored.
If 'mtime' is not None, this will be the file's last-changed timestamp.
If 'mtime' is None, it will use os.path.getmtime().
The survive_deletion flag is used to mark watchers that should stay active even if the underlying file
disappears. Watchers for old data products are created with this flag.
"""
QObject.__init__(self);
self.path = path;
self.enabled = True;
self.quiet = quiet;
dprintf(3,"creating WatchedFile %s, mtime %s (%f)\n",self.path,time.strftime("%x %X",time.localtime(mtime or self.getmtime())),mtime or 0);
self.mtime = mtime or self.getmtime();
self.survive_deletion = survive_deletion;
self.disappeared = False;
def isUpdated(self):
"""Checks if file was updated (i.e. mtime changed) since last check. Returns True if so.
Returns None on access error."""
if not self.enabled:
return None
mtime = self.getmtime()
if mtime is None:
return None
# clear disappeared flag
self.disappeared = False
# compare m,times -- add .1 sec margin since float numbers may get clobbered during
# conversion
updated = (mtime or 0) > (self.mtime or 0) + .1
if updated:
dprintf(4, "WatchedFile %s is updated: mtime %f %s, old mtime %f %s\n", self.path,
mtime, time.strftime("%x %X", time.localtime(mtime)),
self.mtime, time.strftime("%x %X", time.localtime(self.mtime)))
self.mtime = mtime
return updated
def loadPlugins(paths):
global bad_plugins
bad_plugins = {}
# find potential plugin files
modfiles = set()
for path in paths:
files = set(fnmatch.filter(os.listdir(path), "*.py"))
files.discard('__init__.py')
modfiles.update([os.path.join(path, file) for file in files])
dprintf(1, "%d plugins found\n", len(modfiles))
# try to import them
for fullpath in modfiles:
modfile = os.path.basename(fullpath)
modname, ext = os.path.splitext(modfile)
# try to find the module via the imp mechanism
try:
fp, pathname, desc = imp.find_module(modname, paths)
except:
err = sys.exc_info()[1]
dprintf(1, "Error finding module for plugin %s: %s\n", fullpath,
err)
bad_plugins[modname] = err
continue
# try to import the module
# setattr(modname,"plugin_mtime",os.path.getmtime(pathname))
try:
try:
imp.acquire_lock()
module = imp.load_module('Purr.Plugins.%s' % modname, fp,
pathname, desc)
finally:
imp.release_lock()
if fp:
fp.close()
except:
err = sys.exc_info()[1]
dprintf(0, "Error importing module %s: %s\n", pathname, err)
traceback.print_exc()
bad_plugins[modname] = err
continue
# ok, we have a module, check for vital properties
dprintf(1, "Imported plugin module '%s' from %s\n", modname, pathname)
dprintf(1, "%d renderers now available\n", Purr.Render.numRenderers())
def loadPlugins(paths):
global bad_plugins
bad_plugins = {}
# find potential plugin files
modfiles = set()
for path in paths:
files = set(fnmatch.filter(os.listdir(path), "*.py"))
files.discard('__init__.py')
modfiles.update([os.path.join(path, file) for file in files])
dprintf(1, "%d plugins found\n", len(modfiles))
# try to import them
for fullpath in modfiles:
modfile = os.path.basename(fullpath)
modname, ext = os.path.splitext(modfile)
# try to find the module via the imp mechanism
try:
fp, pathname, desc = imp.find_module(modname, paths)
except:
err = sys.exc_info()[1]
dprintf(1, "Error finding module for plugin %s: %s\n", fullpath, err)
bad_plugins[modname] = err
continue
# try to import the module
# setattr(modname,"plugin_mtime",os.path.getmtime(pathname))
try:
try:
imp.acquire_lock()
module = imp.load_module('Purr.Plugins.%s' % modname, fp, pathname, desc)
finally:
imp.release_lock()
if fp:
fp.close()
except:
err = sys.exc_info()[1]
dprintf(0, "Error importing module %s: %s\n", pathname, err)
traceback.print_exc()
bad_plugins[modname] = err
continue
# ok, we have a module, check for vital properties
dprintf(1, "Imported plugin module '%s' from %s\n", modname, pathname)
dprintf(1, "%d renderers now available\n", Purr.Render.numRenderers())
def updatePoliciesFromEntry(self, entry, new=True):
# populate default policies and renames based on entry list
for dp in entry.dps:
# add default policy
basename = os.path.basename(dp.sourcepath)
self._default_dp_props[basename] = dp.policy, dp.filename, dp.comment
dprintf(4, "file %s: default policy is %s\n", basename, dp.policy)
# make new watchers for non-ignored files
if dp.ignored:
# if ignorelistfile is not set, then we're being called from within
# _attach(), when older log entries are being loaded. No need to write the file then.
if new and self.ignorelistfile and os.path.exists(dp.sourcepath):
try:
open(self.ignorelistfile, 'a').write(
"%d %s %s\n" % (os.path.getmtime(dp.sourcepath), dp.policy, dp.sourcepath))
except:
print("Error writing %s" % self.ignorelistfile)
traceback.print_exc()
else:
watcher = self.watchers.get(dp.sourcepath, None)
# if watcher already exists, update timestamp
if watcher:
if watcher.mtime < dp.timestamp:
watcher.mtime = dp.timestamp
dprintf(4, "file %s, updating timestamp to %s\n",
dp.sourcepath, time.strftime("%x %X", time.localtime(dp.timestamp)))
# else create new watcher
else:
wfile = Purrer.WatchedFile(dp.sourcepath, quiet=dp.quiet, mtime=dp.timestamp, survive_deletion=True)
self.watchers[dp.sourcepath] = wfile
dprintf(4, "watching file %s, timestamp %s\n",
dp.sourcepath, time.strftime("%x %X", time.localtime(dp.timestamp)))
def isUpdated(self):
"""Checks if file was updated (i.e. mtime changed) since last check. Returns True if so.
Returns None on access error."""
if not self.enabled:
return None
mtime = self.getmtime()
if mtime is None:
return None
# clear disappeared flag
self.disappeared = False
# compare m,times -- add .1 sec margin since float numbers may get clobbered during
# conversion
updated = (mtime or 0) > (self.mtime or 0) + .1
if updated:
dprintf(
4,
"WatchedFile %s is updated: mtime %f %s, old mtime %f %s\n",
self.path, mtime,
time.strftime("%x %X", time.localtime(mtime)), self.mtime,
time.strftime("%x %X", time.localtime(self.mtime)))
self.mtime = mtime
return updated
def newFiles(self):
"""Returns new files (since last call to newFiles, or since creation).
The only possible new file is the subdirectory itself, which is considered
new if updated, or if a canary has changed.
Return value is an iterable, or None on access error."""
# check directory itself for updates
if self.fileset is None:
return None
# check for new files first
newfiles = Purrer.WatchedDir.newFiles(self)
if newfiles is None:
return None
# this timestamp is assigned to all canaries when directory has changed
timestamp = time.time()
canaries_updated = False
# check for new canaries among new files
if newfiles:
dprintf(3, "directory %s is updated\n", self.path)
for fname in newfiles:
if matches_patterns(os.path.basename(fname),
self.canary_patterns):
self.canaries[fname] = Purrer.WatchedFile(
fname, mtime=timestamp)
dprintf(
3, "watching new canary file %s, timestamp %s\n",
fname,
time.strftime("%x %X", time.localtime(timestamp)))
# else check current canaries for updates
else:
for filename, watcher in list(self.canaries.items()):
updated = watcher.isUpdated()
if updated is None:
dprintf(
2,
"access error on canary %s, will no longer be watched",
filename)
del self.canaries[filename]
continue
elif updated:
dprintf(3, "canary %s is updated\n", filename)
newfiles = True
# this is treated as a bool below
break
# now, if directory has updated, reset timestamps on all canaries
if newfiles:
for watcher in self.canaries.values():
watcher.mtime = timestamp
# returns ourselves (as new file) if something has updated
return (newfiles and [self.path]) or []
def __init__(self,
path,
quiet=None,
mtime=None,
survive_deletion=False):
"""Creates watched file at 'path'. The 'quiet' flag is simply stored.
If 'mtime' is not None, this will be the file's last-changed timestamp.
If 'mtime' is None, it will use os.path.getmtime().
The survive_deletion flag is used to mark watchers that should stay active even if the underlying file
disappears. Watchers for old data products are created with this flag.
"""
QObject.__init__(self)
self.path = path
self.enabled = True
self.quiet = quiet
dprintf(
3, "creating WatchedFile %s, mtime %s (%f)\n", self.path,
time.strftime("%x %X",
time.localtime(mtime or self.getmtime())), mtime
or 0)
self.mtime = mtime or self.getmtime()
self.survive_deletion = survive_deletion
self.disappeared = False
def addWatchedDirectory(self, dirname, watching=Purr.WATCHED, save_config=True):
"""Starts watching the specified directories for changes"""
# see if we're alredy watching this exact set of directories -- do nothing if so
dirname = Purr.canonizePath(dirname)
# do nothing if already watching
if dirname in self.watched_dirs:
dprint(1, "addWatchDirectory(): already watching %s\n", dirname)
# watching=None means do not change the watch-state
if watching is None:
return
else:
if watching is None:
watching = Purr.WATCHED
# make watcher object
wdir = Purrer.WatchedDir(dirname, mtime=self.timestamp,
watch_patterns=self._watch_patterns, ignore_patterns=self._ignore_patterns)
# fileset=None indicates error reading directory, so ignore it
if wdir.fileset is None:
print("There was an error reading the directory %s, will stop watching it." % dirname)
self.setWatchingState(dirname, Purr.REMOVED, save_config=True)
return
self.watchers[dirname] = wdir
self.watched_dirs.append(dirname)
dprintf(2, "watching directory %s, mtime %s, %d files\n",
dirname, time.strftime("%x %X", time.localtime(wdir.mtime)), len(wdir.fileset))
# find files in this directory matching the watch_patterns, and watch them for changes
watchset = set()
for patt in self._watch_patterns:
watchset.update(fnmatch.filter(wdir.fileset, patt))
for fname in watchset:
quiet = matches_patterns(fname, self._quiet_patterns)
fullname = Purr.canonizePath(os.path.join(dirname, fname))
if fullname not in self.watchers:
wfile = Purrer.WatchedFile(fullname, quiet=quiet, mtime=self.timestamp)
self.watchers[fullname] = wfile
dprintf(3, "watching file %s, timestamp %s, quiet %d\n",
fullname, time.strftime("%x %X", time.localtime(wfile.mtime)), quiet)
# find subdirectories matching the subdir_patterns, and watch them for changes
for fname in wdir.fileset:
fullname = Purr.canonizePath(os.path.join(dirname, fname))
if os.path.isdir(fullname):
for desc, dir_patts, canary_patts in self._subdir_patterns:
if matches_patterns(fname, dir_patts):
quiet = matches_patterns(fname, self._quiet_patterns)
wdir = Purrer.WatchedSubdir(fullname, canary_patterns=canary_patts, quiet=quiet,
mtime=self.timestamp)
self.watchers[fullname] = wdir
dprintf(3, "watching subdirectory %s/{%s}, timestamp %s, quiet %d\n",
fullname, ",".join(canary_patts),
time.strftime("%x %X", time.localtime(wdir.mtime)), quiet)
break
# set state and save config
self.setWatchingState(dirname, watching, save_config=save_config)
def newFiles(self):
"""Returns new files (since last call to newFiles, or since creation).
The only possible new file is the subdirectory itself, which is considered
new if updated, or if a canary has changed.
Return value is an iterable, or None on access error."""
# check directory itself for updates
if self.fileset is None:
return None
# check for new files first
newfiles = Purrer.WatchedDir.newFiles(self)
if newfiles is None:
return None
# this timestamp is assigned to all canaries when directory has changed
timestamp = time.time()
canaries_updated = False
# check for new canaries among new files
if newfiles:
dprintf(3, "directory %s is updated\n", self.path)
for fname in newfiles:
if matches_patterns(os.path.basename(fname), self.canary_patterns):
self.canaries[fname] = Purrer.WatchedFile(fname, mtime=timestamp)
dprintf(3, "watching new canary file %s, timestamp %s\n",
fname, time.strftime("%x %X", time.localtime(timestamp)))
# else check current canaries for updates
else:
for filename, watcher in list(self.canaries.items()):
updated = watcher.isUpdated()
if updated is None:
dprintf(2, "access error on canary %s, will no longer be watched", filename)
del self.canaries[filename]
continue
elif updated:
dprintf(3, "canary %s is updated\n", filename)
newfiles = True; # this is treated as a bool below
break
# now, if directory has updated, reset timestamps on all canaries
if newfiles:
for watcher in self.canaries.values():
watcher.mtime = timestamp
# returns ourselves (as new file) if something has updated
return (newfiles and [self.path]) or []
def __init__(self, path, watch_patterns=[], ignore_patterns=[], **kw):
"""Initializes directory.
'ignore_patterns' is a list of patterns to be ignored.
'watch_patterns' is a list of patterns to be watched.
New files will be reported only if they don't match any of the ignore patterns, or
match a watch pattern.
All other arguments as per WatchedFile
"""
Purrer.WatchedFile.__init__(self, path, **kw)
self.watch_patterns = watch_patterns
self.ignore_patterns = ignore_patterns
self._newfiles = []
# the self.fileset attribute gives the current directory content
try:
self.fileset = set(os.listdir(self.path))
except:
_printexc("Error doing listdir(%s)" % self.path)
traceback.print_exc()
self.fileset = None
# this indicates a read error
return
# check for files created after the supplied timestamp
if self.getmtime() > self.mtime:
dprintf(
2,
"%s modified since last run (%f vs %f), checking for new files\n",
self.path, self.getmtime(), self.mtime)
for fname in self.fileset:
# ignore files from ignore list
if matches_patterns(
fname, ignore_patterns) and not matches_patterns(
fname, watch_patterns):
dprintf(
5,
"%s: matches ignore list but not watch list, skipping\n",
fname)
continue
fullname = os.path.join(self.path, fname)
# check creation time against our timestamp
try:
ctime = os.path.getctime(fullname)
except:
_printexc("Error getting ctime for %s, ignoring",
fname)
continue
if ctime > self.mtime:
dprintf(4, "%s: new file (created %s)\n", fullname,
time.strftime("%x %X", time.localtime(ctime)))
# append basename to _newfiles: full path added in newFiles() below
self._newfiles.append(fname)
def load(self, pathname):
"""Loads entry from directory."""
match = self._entry_re.match(pathname)
if not match:
return None
self.ignore = (match.group(1) == "ignore")
if not os.path.isdir(pathname):
raise ValueError("%s: not a directory" % pathname)
if not os.access(pathname, os.R_OK | os.W_OK):
raise ValueError("%s: insufficient access privileges" % pathname)
# parse index.html file
parser = Purr.Parsers.LogEntryIndexParser(pathname)
self.index_file = os.path.join(pathname, 'index.html')
for i, line in enumerate(open(self.index_file)):
try:
parser.feed(line)
except:
dprintf(0, "parse error at line %d of %s\n", i,
self.index_file)
raise
# set things up from parser
try:
self.timestamp = int(float(parser.timestamp))
except:
self.timestamp = int(time.time())
self.title = getattr(parser, 'title', None)
if self.title is None:
self.title = "Malformed entry, probably needs to be deleted"
self.comment = getattr(parser, 'comments', None) or ""
self.dps = getattr(parser, 'dps', [])
self.pathname = pathname
# see if any data products have been removed on us
self.dps = [dp for dp in self.dps if os.path.exists(dp.fullpath)]
# see if the cached include file is up-to-date
self.cached_include = cache = os.path.join(pathname,
'index.include.html')
mtime = (os.path.exists(cache) or 0) and os.path.getmtime(cache)
if mtime >= max(Purr.Render.youngest_renderer,
os.path.getmtime(self.index_file)):
dprintf(2, "entry %s has a valid include cache\n", pathname)
self.cached_include_valid = True
else:
dprintf(2, "entry %s does not have a valid include cache\n",
pathname)
self.cached_include_valid = False
# mark entry as unchanged, if renderers are older than index
self.updated = (Purr.Render.youngest_renderer > os.path.getmtime(
self.index_file))
def load(self, pathname):
"""Loads entry from directory."""
match = self._entry_re.match(pathname)
if not match:
return None
self.ignore = (match.group(1) == "ignore")
if not os.path.isdir(pathname):
raise ValueError("%s: not a directory" % pathname)
if not os.access(pathname, os.R_OK | os.W_OK):
raise ValueError("%s: insufficient access privileges" % pathname)
# parse index.html file
parser = Purr.Parsers.LogEntryIndexParser(pathname)
self.index_file = os.path.join(pathname, 'index.html')
for i, line in enumerate(open(self.index_file)):
try:
parser.feed(line)
except:
dprintf(0, "parse error at line %d of %s\n", i, self.index_file)
raise
# set things up from parser
try:
self.timestamp = int(float(parser.timestamp))
except:
self.timestamp = int(time.time())
self.title = getattr(parser, 'title', None)
if self.title is None:
self.title = "Malformed entry, probably needs to be deleted"
self.comment = getattr(parser, 'comments', None) or ""
self.dps = getattr(parser, 'dps', [])
self.pathname = pathname
# see if any data products have been removed on us
self.dps = [dp for dp in self.dps if os.path.exists(dp.fullpath)]
# see if the cached include file is up-to-date
self.cached_include = cache = os.path.join(pathname, 'index.include.html')
mtime = (os.path.exists(cache) or 0) and os.path.getmtime(cache)
if mtime >= max(Purr.Render.youngest_renderer, os.path.getmtime(self.index_file)):
dprintf(2, "entry %s has a valid include cache\n", pathname)
self.cached_include_valid = True
else:
dprintf(2, "entry %s does not have a valid include cache\n", pathname)
self.cached_include_valid = False
# mark entry as unchanged, if renderers are older than index
self.updated = (Purr.Render.youngest_renderer > os.path.getmtime(self.index_file))
def updatePoliciesFromEntry(self, entry, new=True):
# populate default policies and renames based on entry list
for dp in entry.dps:
# add default policy
basename = os.path.basename(dp.sourcepath)
self._default_dp_props[
basename] = dp.policy, dp.filename, dp.comment
dprintf(4, "file %s: default policy is %s\n", basename, dp.policy)
# make new watchers for non-ignored files
if dp.ignored:
# if ignorelistfile is not set, then we're being called from within
# _attach(), when older log entries are being loaded. No need to write the file then.
if new and self.ignorelistfile and os.path.exists(
dp.sourcepath):
try:
open(self.ignorelistfile,
'a').write("%d %s %s\n" % (os.path.getmtime(
dp.sourcepath), dp.policy, dp.sourcepath))
except:
print("Error writing %s" % self.ignorelistfile)
traceback.print_exc()
else:
watcher = self.watchers.get(dp.sourcepath, None)
# if watcher already exists, update timestamp
if watcher:
if watcher.mtime < dp.timestamp:
watcher.mtime = dp.timestamp
dprintf(
4, "file %s, updating timestamp to %s\n",
dp.sourcepath,
time.strftime("%x %X",
time.localtime(dp.timestamp)))
# else create new watcher
else:
wfile = Purrer.WatchedFile(dp.sourcepath,
quiet=dp.quiet,
mtime=dp.timestamp,
survive_deletion=True)
self.watchers[dp.sourcepath] = wfile
dprintf(
4, "watching file %s, timestamp %s\n", dp.sourcepath,
time.strftime("%x %X", time.localtime(dp.timestamp)))
def __init__(self, path, watch_patterns=[], ignore_patterns=[], **kw):
"""Initializes directory.
'ignore_patterns' is a list of patterns to be ignored.
'watch_patterns' is a list of patterns to be watched.
New files will be reported only if they don't match any of the ignore patterns, or
match a watch pattern.
All other arguments as per WatchedFile
"""
Purrer.WatchedFile.__init__(self, path, **kw)
self.watch_patterns = watch_patterns
self.ignore_patterns = ignore_patterns
self._newfiles = []
# the self.fileset attribute gives the current directory content
try:
self.fileset = set(os.listdir(self.path))
except:
_printexc("Error doing listdir(%s)" % self.path)
traceback.print_exc()
self.fileset = None; # this indicates a read error
return
# check for files created after the supplied timestamp
if self.getmtime() > self.mtime:
dprintf(2, "%s modified since last run (%f vs %f), checking for new files\n", self.path,
self.getmtime(), self.mtime)
for fname in self.fileset:
# ignore files from ignore list
if matches_patterns(fname, ignore_patterns) and not matches_patterns(fname, watch_patterns):
dprintf(5, "%s: matches ignore list but not watch list, skipping\n", fname)
continue
fullname = os.path.join(self.path, fname)
# check creation time against our timestamp
try:
ctime = os.path.getctime(fullname)
except:
_printexc("Error getting ctime for %s, ignoring", fname)
continue
if ctime > self.mtime:
dprintf(4, "%s: new file (created %s)\n", fullname,
time.strftime("%x %X", time.localtime(ctime)))
# append basename to _newfiles: full path added in newFiles() below
self._newfiles.append(fname)
def save(self, dirname=None, refresh=0, refresh_index=True, emit_message=True):
"""Saves entry in the given directory. Data products will be copied over if not
residing in that directory.
'refresh' is a timestamp, passed to renderIndex(), causing all data products OLDER than the specified time to be regenerated.
'refresh_index', if true, causes index files to be re-rendered unconditionally
"""
if not refresh and not self.updated:
return
timestr = time.strftime("%Y%m%d-%H%M%S", time.localtime(self.timestamp))
Purr.progressMessage("Rendering entry for %s" % timestr)
if dirname:
self.pathname = pathname = os.path.join(dirname, "%s-%s" %
(("ignore" if self.ignore else "entry"), timestr))
elif not self.pathname:
raise ValueError("Cannot save entry: pathname not specified")
else:
pathname = self.pathname
# set timestamp
if not self.timestamp:
self.timestamp = int(time.time())
# get canonized path to output directory
pathname = Purr.canonizePath(pathname)
if not os.path.exists(pathname):
os.mkdir(pathname)
# now save content
# get device of pathname -- need to know whether we move or copy
devnum = os.stat(pathname).st_dev
# copy data products as needed
dprintf(2, "saving entry %s, %d data products\n", pathname, len(self.dps))
dps = []
for dp in self.dps:
# if archived, this indicates a previously saved data product, so ignore it
# if ignored, no need to save the DP -- but keep it in list
if dp.archived or dp.ignored:
dprintf(3, "dp %s is archived or ignored, skipping\n", dp.sourcepath)
dps.append(dp)
continue
# file missing for some reason (perhaps it got removed on us?) skip data product entirely
if not os.path.exists(dp.sourcepath):
dprintf(2, "data product %s missing, ignoring\n", dp.sourcepath)
continue
Purr.progressMessage("archiving %s" % dp.filename, sub=True)
# get normalized source and destination paths
dprintf(2, "data product: %s, rename %s, policy %s\n", dp.sourcepath, dp.filename, dp.policy)
sourcepath = Purr.canonizePath(dp.sourcepath)
destname = dp.fullpath = os.path.join(pathname, dp.filename)
dprintf(2, "data product: %s -> %s\n", sourcepath, destname)
# does the destination product already exist? skip if same file, else remove
if os.path.exists(destname):
if os.path.samefile(destname, sourcepath):
dprintf(2, "same file, skipping\n")
dp.timestamp = os.path.getmtime(destname)
dps.append(dp)
continue
if os.system("/bin/rm -fr '%s'" % destname):
print("Error removing %s, which is in the way of %s" % (destname, sourcepath))
print("This data product is not saved.")
continue
# for directories, compress with tar
if os.path.isdir(sourcepath):
sourcepath = sourcepath.rstrip('/')
if dp.policy == "copy" or dp.policy.startswith("move"):
dprintf(2, "archiving to tgz\n")
if os.system("tar zcf '%s' -C '%s' '%s'" % (destname,
os.path.dirname(sourcepath),
os.path.basename(sourcepath))):
print("Error archiving %s to %s" % (sourcepath, destname))
print("This data product is not saved.")
continue
if dp.policy.startswith("move"):
os.system("/bin/rm -fr '%s'" % sourcepath)
# else just a file
else:
# now copy/move it over
if dp.policy == "copy":
dprintf(2, "copying\n")
if _copy_update(sourcepath, destname):
print("Error copying %s to %s" % (sourcepath, destname))
print("This data product is not saved.")
continue
elif dp.policy.startswith('move'):
if _move_update(sourcepath, destname):
print("Error moving %s to %s" % (sourcepath, destname))
print("This data product is not saved.")
continue
# success, set timestamp and append
dp.timestamp = os.path.getmtime(destname)
dp.archived = True
dps.append(dp)
# reset list of data products
self.dps = dps
# now write out content
self.cached_include = os.path.join(pathname, 'index.include.html')
self.cached_include_valid = False
self.index_file = os.path.join(pathname, "index.html")
self.generateIndex(refresh=refresh, refresh_index=refresh_index and time.time())
self.updated = False
def rescan(self):
"""Checks files and directories on watchlist for updates, rescans them for new data products.
If any are found, returns them. Skips those in directories whose watchingState is set to Purr.UNWATCHED.
"""
if not self.attached:
return
dprint(5, "starting rescan")
newstuff = {}
# this accumulates names of new or changed files. Keys are paths, values are 'quiet' flag.
# store timestamp of scan
self.last_scan_timestamp = time.time()
# go through watched files/directories, check for mtime changes
for path, watcher in list(self.watchers.items()):
# get list of new files from watcher
newfiles = watcher.newFiles()
# None indicates access error, so drop it from watcher set
if newfiles is None:
if watcher.survive_deletion:
dprintf(5,
"access error on %s, but will still be watched\n",
watcher.path)
else:
dprintf(2,
"access error on %s, will no longer be watched\n",
watcher.path)
del self.watchers[path]
if not watcher.disappeared:
self.emit(SIGNAL("disappearedFile"), path)
watcher.disappeared = True
continue
dprintf(5, "%s: %d new file(s)\n", watcher.path, len(newfiles))
# if a file has its own watcher, and is independently reported by a directory watcher, skip the directory's
# version and let the file's watcher report it. Reason for this is that the file watcher may have a more
# up-to-date timestamp, so we trust it over the dir watcher.
newfiles = [
p for p in newfiles if p is path or p not in self.watchers
]
# skip files in self._unwatched_paths
newfiles = [
filename for filename in newfiles if self._watching_state.get(
os.path.dirname(filename)) > Purr.UNWATCHED
]
# Now go through files and add them to the newstuff dict
for newfile in newfiles:
# if quiet flag is explicitly set on watcher, enforce it
# if not pouncing on directory, also add quietly
if watcher.quiet or self._watching_state.get(
os.path.dirname(newfile)) < Purr.POUNCE:
quiet = True
# else add quietly if file is not in the quiet patterns
else:
quiet = matches_patterns(os.path.basename(newfile),
self._quiet_patterns)
# add file to list of new products. Since a file may be reported by multiple
# watchers, make the quiet flag a logical AND of all the quiet flags (i.e. DP will be
# marked as quiet only if all watchers report it as quiet).
newstuff[newfile] = quiet and newstuff.get(newfile, True)
dprintf(
4, "%s: new data product, quiet=%d (watcher quiet: %s)\n",
newfile, quiet, watcher.quiet)
# add a watcher for this file to the temp_watchers list. this is used below
# to detect renamed and deleted files
self.temp_watchers[newfile] = Purrer.WatchedFile(newfile)
# now, go through temp_watchers to see if any newly pounced-on files have disappeared
for path, watcher in list(self.temp_watchers.items()):
# get list of new files from watcher
if watcher.newFiles() is None:
dprintf(2, "access error on %s, marking as disappeared",
watcher.path)
del self.temp_watchers[path]
self.emit(SIGNAL("disappearedFile"), path)
# if we have new data products, send them to the main window
return self.makeDataProducts(iter(newstuff.items()))
def addWatchedDirectory(self,
dirname,
watching=Purr.WATCHED,
save_config=True):
"""Starts watching the specified directories for changes"""
# see if we're alredy watching this exact set of directories -- do nothing if so
dirname = Purr.canonizePath(dirname)
# do nothing if already watching
if dirname in self.watched_dirs:
dprint(1, "addWatchDirectory(): already watching %s\n", dirname)
# watching=None means do not change the watch-state
if watching is None:
return
else:
if watching is None:
watching = Purr.WATCHED
# make watcher object
wdir = Purrer.WatchedDir(dirname,
mtime=self.timestamp,
watch_patterns=self._watch_patterns,
ignore_patterns=self._ignore_patterns)
# fileset=None indicates error reading directory, so ignore it
if wdir.fileset is None:
print(
"There was an error reading the directory %s, will stop watching it."
% dirname)
self.setWatchingState(dirname, Purr.REMOVED, save_config=True)
return
self.watchers[dirname] = wdir
self.watched_dirs.append(dirname)
dprintf(2, "watching directory %s, mtime %s, %d files\n", dirname,
time.strftime("%x %X", time.localtime(wdir.mtime)),
len(wdir.fileset))
# find files in this directory matching the watch_patterns, and watch them for changes
watchset = set()
for patt in self._watch_patterns:
watchset.update(fnmatch.filter(wdir.fileset, patt))
for fname in watchset:
quiet = matches_patterns(fname, self._quiet_patterns)
fullname = Purr.canonizePath(os.path.join(dirname, fname))
if fullname not in self.watchers:
wfile = Purrer.WatchedFile(fullname,
quiet=quiet,
mtime=self.timestamp)
self.watchers[fullname] = wfile
dprintf(
3, "watching file %s, timestamp %s, quiet %d\n",
fullname,
time.strftime("%x %X",
time.localtime(wfile.mtime)), quiet)
# find subdirectories matching the subdir_patterns, and watch them for changes
for fname in wdir.fileset:
fullname = Purr.canonizePath(os.path.join(dirname, fname))
if os.path.isdir(fullname):
for desc, dir_patts, canary_patts in self._subdir_patterns:
if matches_patterns(fname, dir_patts):
quiet = matches_patterns(fname,
self._quiet_patterns)
wdir = Purrer.WatchedSubdir(
fullname,
canary_patterns=canary_patts,
quiet=quiet,
mtime=self.timestamp)
self.watchers[fullname] = wdir
dprintf(
3,
"watching subdirectory %s/{%s}, timestamp %s, quiet %d\n",
fullname, ",".join(canary_patts),
time.strftime("%x %X",
time.localtime(wdir.mtime)),
quiet)
break
# set state and save config
self.setWatchingState(dirname, watching, save_config=save_config)
def renderIndex(self, relpath="", refresh=0, refresh_index=0):
"""Returns HTML index code for this entry.
If 'relpath' is empty, renders complete index.html file.
If 'relpath' is not empty, then index is being included into a top-level log, and
relpath should be passed to all sub-renderers.
In this case the entry may make use of its cached_include file, if that is valid.
If 'refresh' is set to a timestamp, then any subproducts (thumbnails, HTML caches, etc.) older than the timestamp will need to be regenerated.
If 'refresh_index' is set to a timestamp, then any index files older than the timestamp will need to be regenerated.
If 'relpath' is empty and 'prev', 'next' and/or 'up' is set, then Prev/Next/Up links will be inserted
"""
# check if cache can be used
refresh_index = max(refresh, refresh_index)
dprintf(2, "%s: rendering HTML index with relpath='%s', refresh=%s refresh_index=%s\n", self.pathname, relpath,
time.strftime("%x %X", time.localtime(refresh)),
time.strftime("%x %X", time.localtime(refresh_index)))
if relpath and self.cached_include_valid:
try:
if os.path.getmtime(self.cached_include) >= refresh_index:
dprintf(2, "using include cache %s\n", self.cached_include)
return open(self.cached_include).read()
else:
dprintf(2, "include cache %s out of date, will regenerate\n", self.cached_include)
self.cached_include_valid = False
except:
print("Error reading cached include code from %s, will regenerate" % self.cached_include)
if verbosity.get_verbose() > 0:
dprint(1, "Error traceback follows:")
traceback.print_exc()
self.cached_include_valid = False
# form up attributes for % operator
attrs = dict(self.__dict__)
attrs['timestr'] = time.strftime("%x %X", time.localtime(self.timestamp))
attrs['relpath'] = relpath
html = ""
# replace title and comments for ignored entries
if self.ignore:
attrs['title'] = "This is not a real log entry"
attrs['comment'] = """This entry was saved by PURR because the user
chose to ignore and/or banish some data products. PURR has stored this
information here for its opwn internal and highly nefarious purposes.
This entry is will not appear in the log."""
# replace < and > in title and comments
attrs['title'] = attrs['title'].replace("<", "<").replace(">", ">")
# write header if asked
if not relpath:
icon = Purr.RenderIndex.renderIcon(24, "..")
html += """<HTML><BODY>
<TITLE>%(title)s</TITLE>""" % attrs
if self._prev_link or self._next_link or self._up_link:
html += """<DIV ALIGN=right><P>%s %s %s</P></DIV>""" % (
(self._prev_link and "<A HREF=\"%s\"><<Previous</A>" % self._prev_link) or "",
(self._up_link and "<A HREF=\"%s\">Up</A>" % self._up_link) or "",
(self._next_link and "<A HREF=\"%s\">Next>></A>" % self._next_link) or ""
)
html += ("<H2>" + icon + """ <A CLASS="TITLE" TIMESTAMP=%(timestamp)d>%(title)s</A></H2>""") % attrs
else:
icon = Purr.RenderIndex.renderIcon(24)
html += """
<HR WIDTH=100%%>
<H2>""" + icon + """ %(title)s</H2>""" % attrs
# write comments
html += """
<DIV ALIGN=right><P><SMALL>Logged on %(timestr)s</SMALL></P></DIV>\n
<A CLASS="COMMENTS">\n""" % attrs
# add comments
logmode = False
for cmt in self.comment.split("\n"):
cmt = cmt.replace("<", "<").replace(">", ">").replace("<BR>", "<BR>")
html += """ <P>%s</P>\n""" % cmt
html += """ </A>\n"""
# add data products
if self.dps:
have_real_dps = bool([dp for dp in self.dps if not dp.ignored])
if have_real_dps:
html += """
<H3>Data products</H3>
<TABLE BORDER=1 FRAME=box RULES=all CELLPADDING=5>\n"""
for dp in self.dps:
dpattrs = dict(dp.__dict__)
dpattrs['comment'] = dpattrs['comment'].replace("<", "<"). \
replace(">", ">").replace('"', "''")
# if generating complete index, write empty anchor for each DP
if not relpath:
if dp.ignored:
html += """
<A CLASS="DP" SRC="%(sourcepath)s" POLICY="%(policy)s" COMMENT="%(comment)s"></A>\n""" % dpattrs
# write normal anchor for normal products
else:
dpattrs['relpath'] = relpath
dpattrs['basename'] = os.path.basename(dp.filename)
html += """
<A CLASS="DP" FILENAME="%(filename)s" SRC="%(sourcepath)s" POLICY="%(policy)s" QUIET=%(quiet)d TIMESTAMP=%(timestamp).6f RENDER="%(render)s" COMMENT="%(comment)s"></A>\n""" % dpattrs
# render a table row
if not dp.ignored:
renderer = Purr.Render.makeRenderer(dp.render, dp, refresh=refresh)
html += Purr.Render.renderInTable(renderer, relpath)
if have_real_dps:
html += """
</TABLE>"""
# write footer
if not relpath:
html += "</BODY></HTML>\n"
else:
# now, write to include cache, if being included
open(self.cached_include, 'w').write(html)
self.cached_include_valid = True
return html
def _attach(self, purrlog, watchdirs=None):
"""Attaches Purr to a purrlog directory, and loads content.
Returns False if nothing new has been loaded (because directory is the same),
or True otherwise."""
purrlog = os.path.abspath(purrlog)
dprint(1, "attaching to purrlog", purrlog)
self.logdir = purrlog
self.indexfile = os.path.join(self.logdir, "index.html")
self.logtitle = "Unnamed log"
self.timestamp = self.last_scan_timestamp = time.time()
self._initIndexDir()
# reset internal state
self.ignorelistfile = None
self.autopounce = False
self.watched_dirs = []
self.entries = []
self._default_dp_props = {}
self.watchers = {}
self.temp_watchers = {}
self.attached = False
self._watching_state = {}
# check that we hold a lock on the directory
self.lockfile = os.path.join(self.logdir, ".purrlock")
# try to open lock file for r/w
try:
self.lockfile_fd = os.open(self.lockfile, os.O_RDWR | os.O_CREAT)
except:
raise Purrer.LockFailError(
"failed to open lock file %s for writing" % self.lockfile)
# try to acquire lock on the lock file
try:
fcntl.lockf(self.lockfile_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except:
other_lock = os.fdopen(self.lockfile_fd, 'r').read()
self.lockfile_fd = None
raise Purrer.LockedError(other_lock)
# got lock, write our ID to the lock file
global _lockstring
try:
self.lockfile_fobj = os.fdopen(self.lockfile_fd, 'w')
self.lockfile_fobj.write(_lockstring)
self.lockfile_fobj.flush()
os.fsync(self.lockfile_fd)
except:
raise
# raise Purrer.LockFailError("cannot write to lock file %s"%self.lockfile)
# load log state if log directory already exists
if os.path.exists(self.logdir):
_busy = Purr.BusyIndicator()
if os.path.exists(self.indexfile):
try:
parser = Purr.Parsers.LogIndexParser()
for line in open(self.indexfile):
parser.feed(line)
self.logtitle = parser.title or self.logtitle
self.timestamp = parser.timestamp or self.timestamp
dprintf(
2, "attached log '%s', timestamp %s\n", self.logtitle,
time.strftime("%x %X", time.localtime(self.timestamp)))
except:
traceback.print_exc()
print("Error parsing %s, reverting to defaults" %
self.indexfile)
# load log entries
entries = []
for fname in os.listdir(self.logdir):
pathname = os.path.join(self.logdir, fname)
if Purr.LogEntry.isValidPathname(pathname):
try:
entry = Purr.LogEntry(load=pathname)
dprint(2, "loaded log entry", pathname)
except:
print("Error loading entry %s, skipping" % fname)
traceback.print_exc()
continue
entries.append(entry)
else:
dprint(2, fname, "is not a valid Purr entry")
# sort log entires by timestamp
entries.sort(lambda a, b: cmp(a.timestamp, b.timestamp))
self.setLogEntries(entries, save=False)
# update own timestamp
if entries:
self.timestamp = max(self.timestamp, entries[-1].timestamp)
# else logfile doesn't exist, create it
else:
self._initIndexDir()
# load configuration if it exists
# init config file
self.dirconfig = configparser.RawConfigParser()
self.dirconfigfile = os.path.join(self.logdir, "dirconfig")
if os.path.exists(self.dirconfigfile):
try:
self.dirconfig.read(self.dirconfigfile)
except:
print("Error loading config file %s" % self.dirconfigfile)
traceback.print_exc()
# load directory configuration
for dirname in self.dirconfig.sections():
try:
watching = self.dirconfig.getint(dirname, "watching")
except:
watching = Purr.WATCHED
dirname = os.path.expanduser(dirname)
self.addWatchedDirectory(dirname, watching, save_config=False)
# start watching the specified directories
for name in (watchdirs or []):
self.addWatchedDirectory(name, watching=None)
# Finally, go through list of ignored files and mark their watchers accordingly.
# The ignorelist is a list of lines of the form "timestamp filename", giving the timestamp when a
# file was last "ignored" by the purrlog user.
self.ignorelistfile = os.path.join(self.logdir, "ignorelist")
if os.path.exists(self.ignorelistfile):
# read lines from file, ignore exceptions
ignores = {}
try:
for line in open(self.ignorelistfile).readlines():
timestamp, policy, filename = line.strip().split(" ", 2)
# update dictiornary with latest timestamp
ignores[filename] = int(timestamp), policy
except:
print("Error reading %s" % self.ignorelistfile)
traceback.print_exc()
# now scan all listed files, and make sure their watchers' mtime is no older than the given
# last-ignore-timestamp. This ensures that we don't pounce on these files after restarting purr.
for filename, (timestamp, policy) in ignores.items():
watcher = self.watchers.get(filename, None)
if watcher:
watcher.mtime = max(watcher.mtime, timestamp)
# init complete
self.attached = True
return True
def save(self,
dirname=None,
refresh=0,
refresh_index=True,
emit_message=True):
"""Saves entry in the given directory. Data products will be copied over if not
residing in that directory.
'refresh' is a timestamp, passed to renderIndex(), causing all data products OLDER than the specified time to be regenerated.
'refresh_index', if true, causes index files to be re-rendered unconditionally
"""
if not refresh and not self.updated:
return
timestr = time.strftime("%Y%m%d-%H%M%S",
time.localtime(self.timestamp))
Purr.progressMessage("Rendering entry for %s" % timestr)
if dirname:
self.pathname = pathname = os.path.join(
dirname,
"%s-%s" % (("ignore" if self.ignore else "entry"), timestr))
elif not self.pathname:
raise ValueError("Cannot save entry: pathname not specified")
else:
pathname = self.pathname
# set timestamp
if not self.timestamp:
self.timestamp = int(time.time())
# get canonized path to output directory
pathname = Purr.canonizePath(pathname)
if not os.path.exists(pathname):
os.mkdir(pathname)
# now save content
# get device of pathname -- need to know whether we move or copy
devnum = os.stat(pathname).st_dev
# copy data products as needed
dprintf(2, "saving entry %s, %d data products\n", pathname,
len(self.dps))
dps = []
for dp in self.dps:
# if archived, this indicates a previously saved data product, so ignore it
# if ignored, no need to save the DP -- but keep it in list
if dp.archived or dp.ignored:
dprintf(3, "dp %s is archived or ignored, skipping\n",
dp.sourcepath)
dps.append(dp)
continue
# file missing for some reason (perhaps it got removed on us?) skip data product entirely
if not os.path.exists(dp.sourcepath):
dprintf(2, "data product %s missing, ignoring\n",
dp.sourcepath)
continue
Purr.progressMessage("archiving %s" % dp.filename, sub=True)
# get normalized source and destination paths
dprintf(2, "data product: %s, rename %s, policy %s\n",
dp.sourcepath, dp.filename, dp.policy)
sourcepath = Purr.canonizePath(dp.sourcepath)
destname = dp.fullpath = os.path.join(pathname, dp.filename)
dprintf(2, "data product: %s -> %s\n", sourcepath, destname)
# does the destination product already exist? skip if same file, else remove
if os.path.exists(destname):
if os.path.samefile(destname, sourcepath):
dprintf(2, "same file, skipping\n")
dp.timestamp = os.path.getmtime(destname)
dps.append(dp)
continue
if os.system("/bin/rm -fr '%s'" % destname):
print("Error removing %s, which is in the way of %s" %
(destname, sourcepath))
print("This data product is not saved.")
continue
# for directories, compress with tar
if os.path.isdir(sourcepath):
sourcepath = sourcepath.rstrip('/')
if dp.policy == "copy" or dp.policy.startswith("move"):
dprintf(2, "archiving to tgz\n")
if os.system("tar zcf '%s' -C '%s' '%s'" %
(destname, os.path.dirname(sourcepath),
os.path.basename(sourcepath))):
print("Error archiving %s to %s" %
(sourcepath, destname))
print("This data product is not saved.")
continue
if dp.policy.startswith("move"):
os.system("/bin/rm -fr '%s'" % sourcepath)
# else just a file
else:
# now copy/move it over
if dp.policy == "copy":
dprintf(2, "copying\n")
if _copy_update(sourcepath, destname):
print("Error copying %s to %s" %
(sourcepath, destname))
print("This data product is not saved.")
continue
elif dp.policy.startswith('move'):
if _move_update(sourcepath, destname):
print("Error moving %s to %s" % (sourcepath, destname))
print("This data product is not saved.")
continue
# success, set timestamp and append
dp.timestamp = os.path.getmtime(destname)
dp.archived = True
dps.append(dp)
# reset list of data products
self.dps = dps
# now write out content
self.cached_include = os.path.join(pathname, 'index.include.html')
self.cached_include_valid = False
self.index_file = os.path.join(pathname, "index.html")
self.generateIndex(refresh=refresh,
refresh_index=refresh_index and time.time())
self.updated = False
def renderIndex(self, relpath="", refresh=0, refresh_index=0):
"""Returns HTML index code for this entry.
If 'relpath' is empty, renders complete index.html file.
If 'relpath' is not empty, then index is being included into a top-level log, and
relpath should be passed to all sub-renderers.
In this case the entry may make use of its cached_include file, if that is valid.
If 'refresh' is set to a timestamp, then any subproducts (thumbnails, HTML caches, etc.) older than the timestamp will need to be regenerated.
If 'refresh_index' is set to a timestamp, then any index files older than the timestamp will need to be regenerated.
If 'relpath' is empty and 'prev', 'next' and/or 'up' is set, then Prev/Next/Up links will be inserted
"""
# check if cache can be used
refresh_index = max(refresh, refresh_index)
dprintf(
2,
"%s: rendering HTML index with relpath='%s', refresh=%s refresh_index=%s\n",
self.pathname, relpath,
time.strftime("%x %X", time.localtime(refresh)),
time.strftime("%x %X", time.localtime(refresh_index)))
if relpath and self.cached_include_valid:
try:
if os.path.getmtime(self.cached_include) >= refresh_index:
dprintf(2, "using include cache %s\n", self.cached_include)
return open(self.cached_include).read()
else:
dprintf(2,
"include cache %s out of date, will regenerate\n",
self.cached_include)
self.cached_include_valid = False
except:
print(
"Error reading cached include code from %s, will regenerate"
% self.cached_include)
if verbosity.get_verbose() > 0:
dprint(1, "Error traceback follows:")
traceback.print_exc()
self.cached_include_valid = False
# form up attributes for % operator
attrs = dict(self.__dict__)
attrs['timestr'] = time.strftime("%x %X",
time.localtime(self.timestamp))
attrs['relpath'] = relpath
html = ""
# replace title and comments for ignored entries
if self.ignore:
attrs['title'] = "This is not a real log entry"
attrs['comment'] = """This entry was saved by PURR because the user
chose to ignore and/or banish some data products. PURR has stored this
information here for its opwn internal and highly nefarious purposes.
This entry is will not appear in the log."""
# replace < and > in title and comments
attrs['title'] = attrs['title'].replace("<",
"<").replace(">", ">")
# write header if asked
if not relpath:
icon = Purr.RenderIndex.renderIcon(24, "..")
html += """<HTML><BODY>
<TITLE>%(title)s</TITLE>""" % attrs
if self._prev_link or self._next_link or self._up_link:
html += """<DIV ALIGN=right><P>%s %s %s</P></DIV>""" % (
(self._prev_link and "<A HREF=\"%s\"><<Previous</A>"
% self._prev_link) or "",
(self._up_link and "<A HREF=\"%s\">Up</A>" % self._up_link)
or "",
(self._next_link and "<A HREF=\"%s\">Next>></A>" %
self._next_link) or "")
html += (
"<H2>" + icon +
""" <A CLASS="TITLE" TIMESTAMP=%(timestamp)d>%(title)s</A></H2>"""
) % attrs
else:
icon = Purr.RenderIndex.renderIcon(24)
html += """
<HR WIDTH=100%%>
<H2>""" + icon + """ %(title)s</H2>""" % attrs
# write comments
html += """
<DIV ALIGN=right><P><SMALL>Logged on %(timestr)s</SMALL></P></DIV>\n
<A CLASS="COMMENTS">\n""" % attrs
# add comments
logmode = False
for cmt in self.comment.split("\n"):
cmt = cmt.replace("<", "<").replace(">", ">").replace(
"<BR>", "<BR>")
html += """ <P>%s</P>\n""" % cmt
html += """ </A>\n"""
# add data products
if self.dps:
have_real_dps = bool([dp for dp in self.dps if not dp.ignored])
if have_real_dps:
html += """
<H3>Data products</H3>
<TABLE BORDER=1 FRAME=box RULES=all CELLPADDING=5>\n"""
for dp in self.dps:
dpattrs = dict(dp.__dict__)
dpattrs['comment'] = dpattrs['comment'].replace("<", "<"). \
replace(">", ">").replace('"', "''")
# if generating complete index, write empty anchor for each DP
if not relpath:
if dp.ignored:
html += """
<A CLASS="DP" SRC="%(sourcepath)s" POLICY="%(policy)s" COMMENT="%(comment)s"></A>\n""" % dpattrs
# write normal anchor for normal products
else:
dpattrs['relpath'] = relpath
dpattrs['basename'] = os.path.basename(dp.filename)
html += """
<A CLASS="DP" FILENAME="%(filename)s" SRC="%(sourcepath)s" POLICY="%(policy)s" QUIET=%(quiet)d TIMESTAMP=%(timestamp).6f RENDER="%(render)s" COMMENT="%(comment)s"></A>\n""" % dpattrs
# render a table row
if not dp.ignored:
renderer = Purr.Render.makeRenderer(dp.render,
dp,
refresh=refresh)
html += Purr.Render.renderInTable(renderer, relpath)
if have_real_dps:
html += """
</TABLE>"""
# write footer
if not relpath:
html += "</BODY></HTML>\n"
else:
# now, write to include cache, if being included
open(self.cached_include, 'w').write(html)
self.cached_include_valid = True
return html
def _attach(self, purrlog, watchdirs=None):
"""Attaches Purr to a purrlog directory, and loads content.
Returns False if nothing new has been loaded (because directory is the same),
or True otherwise."""
purrlog = os.path.abspath(purrlog)
dprint(1, "attaching to purrlog", purrlog)
self.logdir = purrlog
self.indexfile = os.path.join(self.logdir, "index.html")
self.logtitle = "Unnamed log"
self.timestamp = self.last_scan_timestamp = time.time()
self._initIndexDir()
# reset internal state
self.ignorelistfile = None
self.autopounce = False
self.watched_dirs = []
self.entries = []
self._default_dp_props = {}
self.watchers = {}
self.temp_watchers = {}
self.attached = False
self._watching_state = {}
# check that we hold a lock on the directory
self.lockfile = os.path.join(self.logdir, ".purrlock")
# try to open lock file for r/w
try:
self.lockfile_fd = os.open(self.lockfile, os.O_RDWR | os.O_CREAT)
except:
raise Purrer.LockFailError("failed to open lock file %s for writing" % self.lockfile)
# try to acquire lock on the lock file
try:
fcntl.lockf(self.lockfile_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except:
other_lock = os.fdopen(self.lockfile_fd, 'r').read()
self.lockfile_fd = None
raise Purrer.LockedError(other_lock)
# got lock, write our ID to the lock file
global _lockstring
try:
self.lockfile_fobj = os.fdopen(self.lockfile_fd, 'w')
self.lockfile_fobj.write(_lockstring)
self.lockfile_fobj.flush()
os.fsync(self.lockfile_fd)
except:
raise
# raise Purrer.LockFailError("cannot write to lock file %s"%self.lockfile)
# load log state if log directory already exists
if os.path.exists(self.logdir):
_busy = Purr.BusyIndicator()
if os.path.exists(self.indexfile):
try:
parser = Purr.Parsers.LogIndexParser()
for line in open(self.indexfile):
parser.feed(line)
self.logtitle = parser.title or self.logtitle
self.timestamp = parser.timestamp or self.timestamp
dprintf(2, "attached log '%s', timestamp %s\n",
self.logtitle, time.strftime("%x %X", time.localtime(self.timestamp)))
except:
traceback.print_exc()
print("Error parsing %s, reverting to defaults" % self.indexfile)
# load log entries
entries = []
for fname in os.listdir(self.logdir):
pathname = os.path.join(self.logdir, fname)
if Purr.LogEntry.isValidPathname(pathname):
try:
entry = Purr.LogEntry(load=pathname)
dprint(2, "loaded log entry", pathname)
except:
print("Error loading entry %s, skipping" % fname)
traceback.print_exc()
continue
entries.append(entry)
else:
dprint(2, fname, "is not a valid Purr entry")
# sort log entires by timestamp
entries.sort(lambda a, b: cmp(a.timestamp, b.timestamp))
self.setLogEntries(entries, save=False)
# update own timestamp
if entries:
self.timestamp = max(self.timestamp, entries[-1].timestamp)
# else logfile doesn't exist, create it
else:
self._initIndexDir()
# load configuration if it exists
# init config file
self.dirconfig = configparser.RawConfigParser()
self.dirconfigfile = os.path.join(self.logdir, "dirconfig")
if os.path.exists(self.dirconfigfile):
try:
self.dirconfig.read(self.dirconfigfile)
except:
print("Error loading config file %s" % self.dirconfigfile)
traceback.print_exc()
# load directory configuration
for dirname in self.dirconfig.sections():
try:
watching = self.dirconfig.getint(dirname, "watching")
except:
watching = Purr.WATCHED
dirname = os.path.expanduser(dirname)
self.addWatchedDirectory(dirname, watching, save_config=False)
# start watching the specified directories
for name in (watchdirs or []):
self.addWatchedDirectory(name, watching=None)
# Finally, go through list of ignored files and mark their watchers accordingly.
# The ignorelist is a list of lines of the form "timestamp filename", giving the timestamp when a
# file was last "ignored" by the purrlog user.
self.ignorelistfile = os.path.join(self.logdir, "ignorelist")
if os.path.exists(self.ignorelistfile):
# read lines from file, ignore exceptions
ignores = {}
try:
for line in open(self.ignorelistfile).readlines():
timestamp, policy, filename = line.strip().split(" ", 2)
# update dictiornary with latest timestamp
ignores[filename] = int(timestamp), policy
except:
print("Error reading %s" % self.ignorelistfile)
traceback.print_exc()
# now scan all listed files, and make sure their watchers' mtime is no older than the given
# last-ignore-timestamp. This ensures that we don't pounce on these files after restarting purr.
for filename, (timestamp, policy) in ignores.items():
watcher = self.watchers.get(filename, None)
if watcher:
watcher.mtime = max(watcher.mtime, timestamp)
# init complete
self.attached = True
return True
