class Slurp:
def __init__(self, paths, server, name, progress=True):
self.paths = paths
self.current_files = 0
self.current_dir = ''
self.db = Database(server, name)
self.progress = progress
if progress:
self._count_files(paths)
widgets = [Fraction(), ", ", Percentage(), " ", Bar()] if progress else []
self.bar = ProgressBar(self.total_files, widgets=widgets)
def _count_files(self, paths):
self.total_files = 0
for files in self._walk(paths):
self.total_files += len(files)
if self.progress:
sys.stderr.write("Counting files... %d\r" % self.total_files)
if self.progress:
sys.stderr.write("\n")
def run(self):
if self.progress: self.bar.start()
for paths in self._walk(self.paths):
self.current_dir = os.path.dirname(paths[0])
self._update()
try: self.db.add_many(paths)
except Exception:
sys.stderr.write("Error when importing %s:\n" % self.current_dir)
traceback.print_exc()
if self.progress: sys.stderr.write("\n\n")
self._update(paths)
if self.progress: self.bar.finish()
def _update(self, paths = []):
if not self.progress: return
self.bar.fd.write("\033[1A\033[KSlurping %s...\r\033[1B" % self.current_dir)
self.current_files += len(paths)
self.bar.update(self.current_files)
def _walk(self, paths):
for path in paths:
if os.path.isfile(path): yield [os.path.abspath(path)]
else:
for (dirname, _, files) in os.walk(path):
if files:
yield [os.path.abspath(os.path.join(dirname, f)) for f in files]
def __init__(self, paths, server, name, progress=True):
self.paths = paths
self.current_files = 0
self.current_dir = ''
self.db = Database(server, name)
self.progress = progress
if progress:
self._count_files(paths)
widgets = [Fraction(), ", ", Percentage(), " ", Bar()] if progress else []
self.bar = ProgressBar(self.total_files, widgets=widgets)
def __init__(self, server, name, progress=True):
self.current_files = 0
self.db = Database(server, name)
self.progress = progress
self.view = self.db.db.view('_view/update/check')
if self.progress:
sys.stderr.write("Counting files...\r")
self.total_files = len(self.view)
self.bar = ProgressBar(self.total_files)
else:
self.total_files = sys.maxint
class Check:
def __init__(self, server, name, progress=True):
self.current_files = 0
self.db = Database(server, name)
self.progress = progress
self.view = self.db.db.view('_view/update/check')
if self.progress:
sys.stderr.write("Counting files...\r")
self.total_files = len(self.view)
self.bar = ProgressBar(self.total_files)
else:
self.total_files = sys.maxint
def run(self):
if self.progress: self.bar.start()
mount_errors = set()
for i, doc in enumerate(row.value for row in self.view):
if "mountpoint" in doc:
mountpoint = util.qencode(doc["mountpoint"])
if not os.path.ismount(mountpoint):
if not mountpoint in mount_errors:
sys.stderr.write("Error: %s isn't mounted\n" % mountpoint)
if self.progress: sys.stderr.write("\n\n")
mount_errors.add(mountpoint)
continue
filename = util.qencode(doc["filename"])
try:
if not os.path.isfile(filename):
self.db.remove(filename)
elif util.mtime(filename) > doc["mtime"]:
self.db.update(doc, filename)
except Exception:
print "Error when updating %s:" % filename
traceback.print_exc()
if self.progress: sys.stderr.write("\n\n")
if self.progress: self.bar.update(i)
if self.progress: self.bar.finish()
class Process(pyi.ProcessEvent):
def __init__(self, server, name):
self.server = server
self.name = name
self.db = Database(server=server, name=name)
def process_IN_DELETE(self, event):
if event.is_dir: return
path = os.path.join(event.path, event.name)
print "Removing %s..." % path
self.db.remove(path)
def process_IN_MOVED_TO(self, event):
if not event.is_dir:
self.process_default(event)
return
path = os.path.join(event.path, event.name)
print "Slurping %s..." % path
Slurp([path], server=self.server, name=self.name, progress=False).run()
def process_IN_MOVED_FROM(self, event):
if not event.is_dir:
self.process_IN_DELETE(self, event)
path = os.path.join(event.path, event.name)
print "Removing %s..." % path
self.db.remove_docs(self.db.docs_beneath(path))
def process_default(self, event):
if event.is_dir: return
path = os.path.join(event.path, event.name)
print "Updating %s..." % path
for attempt in range(10):
try:
self.db.add(path)
break
except EOFError:
if attempt < 9:
print "File not yet written, waiting a second..."
time.sleep(1)
else:
print "Giving up."
def __init__(self, server, name):
self.server = server
self.name = name
self.db = Database(server=server, name=name)