def lineReceived(self, line):
stdout = self.options.stdout
stderr = self.options.stderr
if self.in_error:
print(quote_output(line, quotemarks=False), file=stderr)
return
if line.startswith("ERROR:"):
self.in_error = True
self.rc = 1
print(quote_output(line, quotemarks=False), file=stderr)
return
try:
d = json.loads(line.decode('utf-8'))
except Exception as e:
print("ERROR could not decode/parse %s\nERROR %r" % (quote_output(line), e), file=stderr)
else:
if d["type"] in ("file", "directory"):
if self.options["storage-index"]:
si = d.get("storage-index", None)
if si:
print(quote_output(si, quotemarks=False), file=stdout)
elif self.options["verify-cap"]:
vc = d.get("verifycap", None)
if vc:
print(quote_output(vc, quotemarks=False), file=stdout)
elif self.options["repair-cap"]:
vc = d.get("repaircap", None)
if vc:
print(quote_output(vc, quotemarks=False), file=stdout)
else:
print("%s %s" % (quote_output(d["cap"], quotemarks=False),
quote_path(d["path"], quotemarks=False)), file=stdout)
def lineReceived(self, line):
if self.in_error:
print >>self.stderr, quote_output(line, quotemarks=False)
return
if line.startswith("ERROR:"):
self.in_error = True
self.streamer.rc = 1
print >>self.stderr, quote_output(line, quotemarks=False)
return
d = simplejson.loads(line)
stdout = self.stdout
if d["type"] not in ("file", "directory"):
return
self.num_objects += 1
# non-verbose means print a progress marker every 100 files
if self.num_objects % 100 == 0:
print >>stdout, "%d objects checked.." % self.num_objects
cr = d["check-results"]
if cr["results"]["healthy"]:
self.files_healthy += 1
else:
self.files_unhealthy += 1
if self.verbose:
# verbose means also print one line per file
path = d["path"]
if not path:
path = ["<root>"]
summary = cr.get("summary", "Healthy (LIT)")
print >>stdout, "%s: %s" % (quote_path(path), quote_output(summary, quotemarks=False))
# always print out corrupt shares
for shareloc in cr["results"].get("list-corrupt-shares", []):
(serverid, storage_index, sharenum) = shareloc
print >>stdout, " corrupt: %s" % _quote_serverid_index_share(serverid, storage_index, sharenum)
def test_quote_path(self):
self.failUnlessReallyEqual(quote_path([u'foo', u'bar']), "'foo/bar'")
self.failUnlessReallyEqual(
quote_path([u'foo', u'bar'], quotemarks=True), "'foo/bar'")
self.failUnlessReallyEqual(
quote_path([u'foo', u'bar'], quotemarks=False), "foo/bar")
self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar']),
'"foo/\\x0abar"')
self.failUnlessReallyEqual(
quote_path([u'foo', u'\nbar'], quotemarks=True), '"foo/\\x0abar"')
self.failUnlessReallyEqual(
quote_path([u'foo', u'\nbar'], quotemarks=False), '"foo/\\x0abar"')
def win32_other(win32, other):
return win32 if sys.platform == "win32" else other
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo"),
win32_other("'C:\\foo'", "'\\\\?\\C:\\foo'"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=True),
win32_other("'C:\\foo'", "'\\\\?\\C:\\foo'"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=False),
win32_other("C:\\foo", "\\\\?\\C:\\foo"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar"),
win32_other("'\\\\foo\\bar'", "'\\\\?\\UNC\\foo\\bar'"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=True),
win32_other("'\\\\foo\\bar'", "'\\\\?\\UNC\\foo\\bar'"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar",
quotemarks=False),
win32_other("\\\\foo\\bar", "\\\\?\\UNC\\foo\\bar"))
def test_quote_path(self):
self.failUnlessReallyEqual(quote_path([u'foo', u'bar']), b"'foo/bar'")
self.failUnlessReallyEqual(
quote_path([u'foo', u'bar'], quotemarks=True), b"'foo/bar'")
self.failUnlessReallyEqual(
quote_path([u'foo', u'bar'], quotemarks=False), b"foo/bar")
self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar']),
b'"foo/\\x0abar"')
self.failUnlessReallyEqual(
quote_path([u'foo', u'\nbar'], quotemarks=True), b'"foo/\\x0abar"')
self.failUnlessReallyEqual(
quote_path([u'foo', u'\nbar'], quotemarks=False),
b'"foo/\\x0abar"')
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo"),
win32_other(b"'C:\\foo'", b"'\\\\?\\C:\\foo'"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=True),
win32_other(b"'C:\\foo'", b"'\\\\?\\C:\\foo'"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=False),
win32_other(b"C:\\foo", b"\\\\?\\C:\\foo"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar"),
win32_other(b"'\\\\foo\\bar'", b"'\\\\?\\UNC\\foo\\bar'"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=True),
win32_other(b"'\\\\foo\\bar'", b"'\\\\?\\UNC\\foo\\bar'"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar",
quotemarks=False),
win32_other(b"\\\\foo\\bar", b"\\\\?\\UNC\\foo\\bar"))
def test_quote_path(self):
self.failUnlessReallyEqual(quote_path([u"foo", u"bar"]), "'foo/bar'")
self.failUnlessReallyEqual(quote_path([u"foo", u"bar"], quotemarks=True), "'foo/bar'")
self.failUnlessReallyEqual(quote_path([u"foo", u"bar"], quotemarks=False), "foo/bar")
self.failUnlessReallyEqual(quote_path([u"foo", u"\nbar"]), '"foo/\\x0abar"')
self.failUnlessReallyEqual(quote_path([u"foo", u"\nbar"], quotemarks=True), '"foo/\\x0abar"')
self.failUnlessReallyEqual(quote_path([u"foo", u"\nbar"], quotemarks=False), '"foo/\\x0abar"')
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo"), win32_other("'C:\\foo'", "'\\\\?\\C:\\foo'")
)
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=True), win32_other("'C:\\foo'", "'\\\\?\\C:\\foo'")
)
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=False), win32_other("C:\\foo", "\\\\?\\C:\\foo")
)
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar"), win32_other("'\\\\foo\\bar'", "'\\\\?\\UNC\\foo\\bar'")
)
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=True),
win32_other("'\\\\foo\\bar'", "'\\\\?\\UNC\\foo\\bar'"),
)
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=False),
win32_other("\\\\foo\\bar", "\\\\?\\UNC\\foo\\bar"),
)
def lineReceived(self, line):
if self.in_error:
print(quote_output(line, quotemarks=False), file=self.stderr)
return
if line.startswith("ERROR:"):
self.in_error = True
self.streamer.rc = 1
print(quote_output(line, quotemarks=False), file=self.stderr)
return
d = json.loads(line)
stdout = self.stdout
if d["type"] not in ("file", "directory"):
return
self.num_objects += 1
# non-verbose means print a progress marker every 100 files
if self.num_objects % 100 == 0:
print("%d objects checked.." % self.num_objects, file=stdout)
cr = d["check-results"]
if cr["results"]["healthy"]:
self.files_healthy += 1
else:
self.files_unhealthy += 1
if self.verbose:
# verbose means also print one line per file
path = d["path"]
if not path:
path = ["<root>"]
# LIT files and directories do not have a "summary" field.
summary = cr.get("summary", "Healthy (LIT)")
print("%s: %s" %
(quote_path(path), quote_output(summary, quotemarks=False)),
file=stdout)
# always print out corrupt shares
for shareloc in cr["results"].get("list-corrupt-shares", []):
(serverid, storage_index, sharenum) = shareloc
print(
" corrupt: %s" %
_quote_serverid_index_share(serverid, storage_index, sharenum),
file=stdout)
def lineReceived(self, line):
stdout = self.options.stdout
stderr = self.options.stderr
if self.in_error:
print(quote_output(line, quotemarks=False), file=stderr)
return
if line.startswith(b"ERROR:"):
self.in_error = True
self.rc = 1
print(quote_output(line, quotemarks=False), file=stderr)
return
try:
d = json.loads(line.decode('utf-8'))
except Exception as e:
print("ERROR could not decode/parse %s\nERROR %r" %
(quote_output(line), e),
file=stderr)
else:
if d["type"] in ("file", "directory"):
if self.options["storage-index"]:
si = d.get("storage-index", None)
if si:
print(quote_output(si, quotemarks=False), file=stdout)
elif self.options["verify-cap"]:
vc = d.get("verifycap", None)
if vc:
print(quote_output(vc, quotemarks=False), file=stdout)
elif self.options["repair-cap"]:
vc = d.get("repaircap", None)
if vc:
print(quote_output(vc, quotemarks=False), file=stdout)
else:
# ensure_str() only necessary for Python 2.
print(ensure_str("%s %s") %
(quote_output(d["cap"], quotemarks=False),
quote_path(d["path"], quotemarks=False)),
file=stdout)
def test_quote_path(self):
self.failUnlessReallyEqual(quote_path([u'foo', u'bar']), "'foo/bar'")
self.failUnlessReallyEqual(quote_path([u'foo', u'bar'], quotemarks=True), "'foo/bar'")
self.failUnlessReallyEqual(quote_path([u'foo', u'bar'], quotemarks=False), "foo/bar")
self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar']), '"foo/\\x0abar"')
self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar'], quotemarks=True), '"foo/\\x0abar"')
self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar'], quotemarks=False), '"foo/\\x0abar"')
def win32_other(win32, other):
return win32 if sys.platform == "win32" else other
self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo"),
win32_other("'C:\\foo'", "'\\\\?\\C:\\foo'"))
self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=True),
win32_other("'C:\\foo'", "'\\\\?\\C:\\foo'"))
self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=False),
win32_other("C:\\foo", "\\\\?\\C:\\foo"))
self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar"),
win32_other("'\\\\foo\\bar'", "'\\\\?\\UNC\\foo\\bar'"))
self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=True),
win32_other("'\\\\foo\\bar'", "'\\\\?\\UNC\\foo\\bar'"))
self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=False),
win32_other("\\\\foo\\bar", "\\\\?\\UNC\\foo\\bar"))
def lineReceived(self, line):
if self.in_error:
print >> self.stderr, quote_output(line, quotemarks=False)
return
if line.startswith("ERROR:"):
self.in_error = True
self.streamer.rc = 1
print >> self.stderr, quote_output(line, quotemarks=False)
return
d = simplejson.loads(line)
stdout = self.stdout
if d["type"] not in ("file", "directory"):
return
self.num_objects += 1
# non-verbose means print a progress marker every 100 files
if self.num_objects % 100 == 0:
print >> stdout, "%d objects checked.." % self.num_objects
crr = d["check-and-repair-results"]
if d["storage-index"]:
if crr["pre-repair-results"]["results"]["healthy"]:
was_healthy = True
self.pre_repair_files_healthy += 1
else:
was_healthy = False
self.pre_repair_files_unhealthy += 1
if crr["post-repair-results"]["results"]["healthy"]:
self.post_repair_files_healthy += 1
else:
self.post_repair_files_unhealthy += 1
else:
# LIT file
was_healthy = True
self.pre_repair_files_healthy += 1
self.post_repair_files_healthy += 1
if crr["repair-attempted"]:
self.repairs_attempted += 1
if crr["repair-successful"]:
self.repairs_successful += 1
if self.verbose:
# verbose means also print one line per file
path = d["path"]
if not path:
path = ["<root>"]
# we don't seem to have a summary available, so build one
if was_healthy:
summary = "healthy"
else:
summary = "not healthy"
print >> stdout, "%s: %s" % (quote_path(path), summary)
# always print out corrupt shares
prr = crr.get("pre-repair-results", {})
for shareloc in prr.get("results", {}).get("list-corrupt-shares", []):
(serverid, storage_index, sharenum) = shareloc
print >> stdout, " corrupt: %s" % _quote_serverid_index_share(
serverid, storage_index, sharenum)
# always print out repairs
if crr["repair-attempted"]:
if crr["repair-successful"]:
print >> stdout, " repair successful"
else:
print >> stdout, " repair failed"
def lineReceived(self, line):
if self.in_error:
print(quote_output(line, quotemarks=False), file=self.stderr)
return
if line.startswith("ERROR:"):
self.in_error = True
self.streamer.rc = 1
print(quote_output(line, quotemarks=False), file=self.stderr)
return
d = json.loads(line)
stdout = self.stdout
if d["type"] not in ("file", "directory"):
return
self.num_objects += 1
# non-verbose means print a progress marker every 100 files
if self.num_objects % 100 == 0:
print("%d objects checked.." % self.num_objects, file=stdout)
crr = d["check-and-repair-results"]
if d["storage-index"]:
if crr["pre-repair-results"]["results"]["healthy"]:
was_healthy = True
self.pre_repair_files_healthy += 1
else:
was_healthy = False
self.pre_repair_files_unhealthy += 1
if crr["post-repair-results"]["results"]["healthy"]:
self.post_repair_files_healthy += 1
else:
self.post_repair_files_unhealthy += 1
else:
# LIT file
was_healthy = True
self.pre_repair_files_healthy += 1
self.post_repair_files_healthy += 1
if crr["repair-attempted"]:
self.repairs_attempted += 1
if crr["repair-successful"]:
self.repairs_successful += 1
if self.verbose:
# verbose means also print one line per file
path = d["path"]
if not path:
path = ["<root>"]
# we don't seem to have a summary available, so build one
if was_healthy:
summary = "healthy"
else:
summary = "not healthy"
print("%s: %s" % (quote_path(path), summary), file=stdout)
# always print out corrupt shares
prr = crr.get("pre-repair-results", {})
for shareloc in prr.get("results", {}).get("list-corrupt-shares", []):
(serverid, storage_index, sharenum) = shareloc
print(" corrupt: %s" % _quote_serverid_index_share(serverid, storage_index, sharenum), file=stdout)
# always print out repairs
if crr["repair-attempted"]:
if crr["repair-successful"]:
print(" repair successful", file=stdout)
else:
print(" repair failed", file=stdout)
if d["type"] in ("file", "directory"):
if self.options["storage-index"]:
si = d.get("storage-index", None)
if si:
print >>stdout, quote_output(si, quotemarks=False)
elif self.options["verify-cap"]:
vc = d.get("verifycap", None)
if vc:
print >>stdout, quote_output(vc, quotemarks=False)
elif self.options["repair-cap"]:
vc = d.get("repaircap", None)
if vc:
print >>stdout, quote_output(vc, quotemarks=False)
else:
print >>stdout, "%s %s" % (quote_output(d["cap"], quotemarks=False),
quote_path(d["path"], quotemarks=False))
def manifest(options):
return ManifestStreamer().run(options)
class StatsGrabber(SlowOperationRunner):
def make_url(self, base, ophandle):
return base + "?t=start-deep-stats&ophandle=" + ophandle
def write_results(self, data):
stdout = self.options.stdout
keys = ("count-immutable-files",
"count-mutable-files",
"count-literal-files",
"count-files",
if d["type"] in ("file", "directory"):
if self.options["storage-index"]:
si = d.get("storage-index", None)
if si:
print >>stdout, quote_output(si, quotemarks=False)
elif self.options["verify-cap"]:
vc = d.get("verifycap", None)
if vc:
print >>stdout, quote_output(vc, quotemarks=False)
elif self.options["repair-cap"]:
vc = d.get("repaircap", None)
if vc:
print >>stdout, quote_output(vc, quotemarks=False)
else:
print >>stdout, "%s %s" % (quote_output(d["cap"], quotemarks=False),
quote_path(d["path"], quotemarks=False))
def manifest(options):
return ManifestStreamer().run(options)
class StatsGrabber(SlowOperationRunner):
def make_url(self, base, ophandle):
return base + "?t=start-deep-stats&ophandle=" + ophandle
def write_results(self, data):
stdout = self.options.stdout
keys = ("count-immutable-files",
"count-mutable-files",
"count-literal-files",
"count-files",
