def delete_extra_files(index_by_cached_names, res_folder):
extras = 0
scanned = 0
extra_files = []
for root, dirs, files in os.walk(res_folder):
for name in files:
scanned += 1
progress.write("Scanned %7.1d files (%7.1d extra)\r" % (scanned, extras))
parent_folder = os.path.split(root)[1]
cached_name = "%s/%s" % (parent_folder, name)
if cached_name not in index_by_cached_names:
extras += 1
extra_files.append(os.path.join(root, name))
progress.clear()
removed = 0
for filename in extra_files:
try:
progress.write("Removed %7.1d of %7.1d files\r" % (removed, extras))
os.remove(filename)
removed += 1
except IOError:
pass
return extras
def verify_cache(index, res_folder):
num_files = len(index)
missing = 0
corrupt = 0
scanned = 0
for entry in index:
scanned += 1
progress.write("Scanned %6.1d of %6.1d files (%6.1d corrupt, %6.1d missing)\r" %
(scanned, num_files, corrupt, missing))
filename = os.path.join(res_folder, entry.cached_name)
if not os.path.exists(filename):
missing += 1
continue
checksum = calc_checksum(filename)
if checksum != entry.md5_checksum:
corrupt += 1
try:
os.remove(filename)
except IOError:
pass
progress.clear()
print "Verified %d files:" % num_files
print " %6.1d files corrupt" % corrupt
print " %6.1d files not yet downloaded" % missing
return corrupt, missing
def scan_missing_files(index, res_folder):
num_files = len(index)
missing = 0
scanned = 0
missing_files = []
missing_bytes = 0
missing_bytes_on_disk = 0
for entry in index:
scanned += 1
progress.write(
"%6.1d of %6.1d files (%6.1d missing - %10.10s - %10.10s on disk)\r"
% (scanned, num_files, missing, format_memory(missing_bytes),
format_memory(missing_bytes_on_disk)))
filename = os.path.join(res_folder, entry.cached_name)
if not os.path.exists(filename):
missing += 1
missing_files.append(entry)
missing_bytes += entry.compressed_size
missing_bytes_on_disk += entry.size_in_bytes
progress.clear()
print "%6.1d files missing - %10.10s - %10.10s on disk\r" % \
(missing, format_memory(missing_bytes), format_memory(missing_bytes_on_disk))
print
return missing_files
def process(path, prefix=None):
for entry in os.listdir(path):
entry_path = os.path.join(path, entry)
branch_name = os.path.join(prefix, entry) if prefix else entry
if os.path.isdir(entry_path):
process(entry_path, branch_name)
elif not exists_in_db(branch_name):
progress.write("WARNING[%s]: %s exists in the repository but not in the database!" % (repository.name, branch_name))
if force: repository.run("update-ref", "-d", "refs/heads/%s" % branch_name)
progress.write(" deleted from repository")
processed.add(branch_name)
def download_missing_files(res_folder, files_to_download):
q = Queue.Queue()
for f in files_to_download:
q.put(f)
downloaded_files = 0
old_size = q.qsize()
num_files = len(files_to_download)
thread_list = []
for i in range(DOWNLOAD_THREAD_COUNT):
t = DownloadThread(res_folder, q)
thread_list.append(t)
t.start()
try:
while not q.empty():
progress.write("Downloaded %6.1d of %6.1d files\r" %
(downloaded_files, num_files))
new_size = q.qsize()
d_size = old_size - new_size
if d_size:
downloaded_files += d_size
old_size = new_size
time.sleep(0.5)
progress.clear()
num_failed = 0
num_succeeded = 0
for t in thread_list:
t.join()
num_failed += t.failed
num_succeeded += t.succeeded
if t.messages:
print t.messages
return num_succeeded, num_failed
except KeyboardInterrupt:
progress.clear()
print "Stopping download threads"
for t in thread_list:
t.stop()
for t in thread_list:
t.join()
raise
def download_missing_files(res_folder, files_to_download):
q = Queue.Queue()
for f in files_to_download:
q.put(f)
downloaded_files = 0
old_size = q.qsize()
num_files = len(files_to_download)
thread_list = []
for i in range(DOWNLOAD_THREAD_COUNT):
t = DownloadThread(res_folder, q)
thread_list.append(t)
t.start()
try:
while not q.empty():
progress.write("Downloaded %6.1d of %6.1d files\r" % (downloaded_files, num_files))
new_size = q.qsize()
d_size = old_size - new_size
if d_size:
downloaded_files += d_size
old_size = new_size
time.sleep(0.5)
progress.clear()
num_failed = 0
num_succeeded = 0
for t in thread_list:
t.join()
num_failed += t.failed
num_succeeded += t.succeeded
if t.messages:
print t.messages
return num_succeeded, num_failed
except KeyboardInterrupt:
progress.clear()
print "Stopping download threads"
for t in thread_list:
t.stop()
for t in thread_list:
t.join()
raise
def scan_extra_files(index_by_cached_names, res_folder):
print "Scanning %s, checking for extra files" % res_folder
extras = 0
extra_bytes = 0
scanned = 0
for root, dirs, files in os.walk(res_folder):
for name in files:
scanned += 1
progress.write("Scanned %7.1d files (%7.1d extra)\r" % (scanned, extras))
parent_folder = os.path.split(root)[1]
cached_name = "%s/%s" % (parent_folder, name)
if cached_name not in index_by_cached_names:
extras += 1
full_name = os.path.join(root, name)
extra_bytes += os.path.getsize(full_name)
progress.clear()
return extras, extra_bytes
def scan_extra_files(index_by_cached_names, res_folder):
print "Scanning %s, checking for extra files" % res_folder
extras = 0
extra_bytes = 0
scanned = 0
for root, dirs, files in os.walk(res_folder):
for name in files:
scanned += 1
progress.write("Scanned %7.1d files (%7.1d extra)\r" %
(scanned, extras))
parent_folder = os.path.split(root)[1]
cached_name = "%s/%s" % (parent_folder, name)
if cached_name not in index_by_cached_names:
extras += 1
full_name = os.path.join(root, name)
extra_bytes += os.path.getsize(full_name)
progress.clear()
return extras, extra_bytes
def scan_missing_files(index, res_folder):
print "Scanning index, checking for missing files"
num_files = len(index)
missing = 0
missing_bytes = 0
missing_download_bytes = 0
scanned = 0
for entry in index:
scanned += 1
progress.write("Scanned %6.1d of %6.1d files (%6.1d missing)\r" % (scanned, num_files, missing))
filename = os.path.join(res_folder, entry.cached_name)
if not os.path.exists(filename):
missing += 1
missing_download_bytes += entry.compressed_size
missing_bytes += entry.size_in_bytes
continue
progress.clear()
return missing, missing_bytes, missing_download_bytes
def scan_missing_files(index, res_folder):
print "Scanning index, checking for missing files"
num_files = len(index)
missing = 0
missing_bytes = 0
missing_download_bytes = 0
scanned = 0
for entry in index:
scanned += 1
progress.write("Scanned %6.1d of %6.1d files (%6.1d missing)\r" %
(scanned, num_files, missing))
filename = os.path.join(res_folder, entry.cached_name)
if not os.path.exists(filename):
missing += 1
missing_download_bytes += entry.compressed_size
missing_bytes += entry.size_in_bytes
continue
progress.clear()
return missing, missing_bytes, missing_download_bytes
def scan_missing_files(index, res_folder):
num_files = len(index)
missing = 0
scanned = 0
missing_files = []
missing_bytes = 0
missing_bytes_on_disk = 0
for entry in index:
scanned += 1
progress.write("%6.1d of %6.1d files (%6.1d missing - %10.10s - %10.10s on disk)\r" %
(scanned, num_files, missing, format_memory(missing_bytes), format_memory(missing_bytes_on_disk)))
filename = os.path.join(res_folder, entry.cached_name)
if not os.path.exists(filename):
missing += 1
missing_files.append(entry)
missing_bytes += entry.compressed_size
missing_bytes_on_disk += entry.size_in_bytes
progress.clear()
print "%6.1d files missing - %10.10s - %10.10s on disk\r" % \
(missing, format_memory(missing_bytes), format_memory(missing_bytes_on_disk))
print
return missing_files
if error.errno == errno.ENOENT: pass
else: raise
progress.start(len(branches), "Repository: %s" % repository.name)
heads_path = os.path.join(repository.path, "refs", "heads")
for branch_id, branch_name, branch_type, branch_base_id, branch_sha1 in branches:
progress.update()
try:
try: repository_sha1 = open(os.path.join(heads_path, branch_name)).read().strip()
except: repository_sha1 = refs.get(branch_name)
if repository_sha1 != branch_sha1:
progress.write("NOTE[%s]: %s differs (db:%s != repo:%s)" % (repository.name, branch_name, branch_sha1[:8], repository_sha1[:8]))
if branch_type == "review":
head = getReviewHead(repository, getReview(branch_id))
if not head:
progress.write(" invalid review meta-data: r/%d" % review_id)
continue
if head.sha1 == branch_sha1:
progress.write(" branches.head matches review meta-data; repository is wrong")
if force: repository.run("update-ref", "refs/heads/%s" % branch_name, head.sha1, repository_sha1)
progress.write(" repository updated")
elif head.sha1 == repository_sha1:
progress.write(" repository matches review meta-data; branches.head is wrong")
if force: cursor.execute("UPDATE branches SET head=%s WHERE id=%s", (head.getId(db), branch_id))
heads_path = os.path.join(repository.path, "refs", "heads")
branches_in_db = set()
for branch_id, branch_name, branch_type, branch_base_id, branch_sha1 in branches:
progress.update()
branches_in_db.add(branch_name)
try:
try: repository_sha1 = open(os.path.join(heads_path, branch_name)).read().strip()
except: repository_sha1 = refs.get(branch_name)
if repository_sha1 != branch_sha1:
progress.write("NOTE[%s]: %s differs (db:%s != repo:%s)" % (repository.name, branch_name, branch_sha1[:8], repository_sha1[:8]))
if branch_type == "review":
head = getReviewHead(repository, getReview(branch_id))
if not head:
progress.write(" invalid review meta-data: r/%d" % getReview(branch_id))
continue
if head.sha1 == branch_sha1:
progress.write(" branches.head matches review meta-data; repository is wrong")
if force: repository.run("update-ref", "refs/heads/%s" % branch_name, head.sha1, repository_sha1)
progress.write(" repository updated")
elif head.sha1 == repository_sha1:
progress.write(" repository matches review meta-data; branches.head is wrong")
if force: cursor.execute("UPDATE branches SET head=%s WHERE id=%s", (head.getId(db), branch_id))
