def delete_extra_files(index_by_cached_names, res_folder):
extras = 0
scanned = 0
extra_files = []
for root, dirs, files in os.walk(res_folder):
for name in files:
scanned += 1
progress.write("Scanned %7.1d files (%7.1d extra)\r" % (scanned, extras))
parent_folder = os.path.split(root)[1]
cached_name = "%s/%s" % (parent_folder, name)
if cached_name not in index_by_cached_names:
extras += 1
extra_files.append(os.path.join(root, name))
progress.clear()
removed = 0
for filename in extra_files:
try:
progress.write("Removed %7.1d of %7.1d files\r" % (removed, extras))
os.remove(filename)
removed += 1
except IOError:
pass
return extras
def resetDeviceState(self):
if self.deviceState.has_key(self.device):
del self.deviceState[self.device]
pushConfig.recycleConfig(self.config)
progress.clear(self.taskId, self.device)
if allTaskProgress.has_key(self.taskId) and allTaskProgress[self.taskId] == self.device:
del allTaskProgress[self.taskId]
def scan_missing_files(index, res_folder):
num_files = len(index)
missing = 0
scanned = 0
missing_files = []
missing_bytes = 0
missing_bytes_on_disk = 0
for entry in index:
scanned += 1
progress.write(
"%6.1d of %6.1d files (%6.1d missing - %10.10s - %10.10s on disk)\r"
% (scanned, num_files, missing, format_memory(missing_bytes),
format_memory(missing_bytes_on_disk)))
filename = os.path.join(res_folder, entry.cached_name)
if not os.path.exists(filename):
missing += 1
missing_files.append(entry)
missing_bytes += entry.compressed_size
missing_bytes_on_disk += entry.size_in_bytes
progress.clear()
print "%6.1d files missing - %10.10s - %10.10s on disk\r" % \
(missing, format_memory(missing_bytes), format_memory(missing_bytes_on_disk))
print
return missing_files
def verify_cache(index, res_folder):
num_files = len(index)
missing = 0
corrupt = 0
scanned = 0
for entry in index:
scanned += 1
progress.write("Scanned %6.1d of %6.1d files (%6.1d corrupt, %6.1d missing)\r" %
(scanned, num_files, corrupt, missing))
filename = os.path.join(res_folder, entry.cached_name)
if not os.path.exists(filename):
missing += 1
continue
checksum = calc_checksum(filename)
if checksum != entry.md5_checksum:
corrupt += 1
try:
os.remove(filename)
except IOError:
pass
progress.clear()
print "Verified %d files:" % num_files
print " %6.1d files corrupt" % corrupt
print " %6.1d files not yet downloaded" % missing
return corrupt, missing
def main():
progress.stream = sys.stdout
if len(sys.argv) < 2:
run_interactive()
sys.exit()
parser = argparse.ArgumentParser(
description="rescache is a tool for verifying and managing the EVE shared resource cache."
)
parser.add_argument(
"-i", "--index",
default=DEFAULT_INDEX_FILENAME,
help="The name of an index file to use - defaults to %s" % DEFAULT_INDEX_FILENAME
)
parser.add_argument(
"-c", "--cache",
default=get_shared_cache_folder(),
help="The location of the shared cache to use - defaults to what the EVE client uses"
)
parser.add_argument(
"-d", "--dir",
action='append',
help="Additional directories (EVE installs) to read indexes from"
)
subparsers = parser.add_subparsers()
parser_verify = subparsers.add_parser("verify")
parser_verify.set_defaults(func=verify_command)
parser_diff = subparsers.add_parser("diff")
parser_diff.set_defaults(func=diff_command)
parser_purge = subparsers.add_parser("purge")
parser_purge.set_defaults(func=purge_command)
parser_download = subparsers.add_parser("download")
parser_download.set_defaults(func=download_command)
parser_move = subparsers.add_parser("move")
parser_move.add_argument(
"destination",
help="The name of the cache folder. This folder must not exist already."
)
parser_move.set_defaults(func=move_command)
args = parser.parse_args()
try:
args.func(args)
except KeyboardInterrupt:
progress.clear()
print "Operation cancelled"
sys.exit(1)
def main():
progress.stream = sys.stdout
if len(sys.argv) < 2:
run_interactive()
sys.exit()
parser = argparse.ArgumentParser(
description=
"rescache is a tool for verifying and managing the EVE shared resource cache."
)
parser.add_argument(
"-i",
"--index",
default=DEFAULT_INDEX_FILENAME,
help="The name of an index file to use - defaults to %s" %
DEFAULT_INDEX_FILENAME)
parser.add_argument(
"-c",
"--cache",
default=get_shared_cache_folder(),
help=
"The location of the shared cache to use - defaults to what the EVE client uses"
)
subparsers = parser.add_subparsers()
parser_verify = subparsers.add_parser("verify")
parser_verify.set_defaults(func=verify_command)
parser_diff = subparsers.add_parser("diff")
parser_diff.set_defaults(func=diff_command)
parser_purge = subparsers.add_parser("purge")
parser_purge.set_defaults(func=purge_command)
parser_download = subparsers.add_parser("download")
parser_download.set_defaults(func=download_command)
parser_move = subparsers.add_parser("move")
parser_move.add_argument(
"destination",
help="The name of the cache folder. This folder must not exist already."
)
parser_move.set_defaults(func=move_command)
args = parser.parse_args()
try:
args.func(args)
except KeyboardInterrupt:
progress.clear()
print "Operation cancelled"
sys.exit(1)
def download_missing_files(res_folder, files_to_download):
q = Queue.Queue()
for f in files_to_download:
q.put(f)
downloaded_files = 0
old_size = q.qsize()
num_files = len(files_to_download)
thread_list = []
for i in range(DOWNLOAD_THREAD_COUNT):
t = DownloadThread(res_folder, q)
thread_list.append(t)
t.start()
try:
while not q.empty():
progress.write("Downloaded %6.1d of %6.1d files\r" %
(downloaded_files, num_files))
new_size = q.qsize()
d_size = old_size - new_size
if d_size:
downloaded_files += d_size
old_size = new_size
time.sleep(0.5)
progress.clear()
num_failed = 0
num_succeeded = 0
for t in thread_list:
t.join()
num_failed += t.failed
num_succeeded += t.succeeded
if t.messages:
print t.messages
return num_succeeded, num_failed
except KeyboardInterrupt:
progress.clear()
print "Stopping download threads"
for t in thread_list:
t.stop()
for t in thread_list:
t.join()
raise
def download_missing_files(res_folder, files_to_download):
q = Queue.Queue()
for f in files_to_download:
q.put(f)
downloaded_files = 0
old_size = q.qsize()
num_files = len(files_to_download)
thread_list = []
for i in range(DOWNLOAD_THREAD_COUNT):
t = DownloadThread(res_folder, q)
thread_list.append(t)
t.start()
try:
while not q.empty():
progress.write("Downloaded %6.1d of %6.1d files\r" % (downloaded_files, num_files))
new_size = q.qsize()
d_size = old_size - new_size
if d_size:
downloaded_files += d_size
old_size = new_size
time.sleep(0.5)
progress.clear()
num_failed = 0
num_succeeded = 0
for t in thread_list:
t.join()
num_failed += t.failed
num_succeeded += t.succeeded
if t.messages:
print t.messages
return num_succeeded, num_failed
except KeyboardInterrupt:
progress.clear()
print "Stopping download threads"
for t in thread_list:
t.stop()
for t in thread_list:
t.join()
raise
def scan_extra_files(index_by_cached_names, res_folder):
print "Scanning %s, checking for extra files" % res_folder
extras = 0
extra_bytes = 0
scanned = 0
for root, dirs, files in os.walk(res_folder):
for name in files:
scanned += 1
progress.write("Scanned %7.1d files (%7.1d extra)\r" % (scanned, extras))
parent_folder = os.path.split(root)[1]
cached_name = "%s/%s" % (parent_folder, name)
if cached_name not in index_by_cached_names:
extras += 1
full_name = os.path.join(root, name)
extra_bytes += os.path.getsize(full_name)
progress.clear()
return extras, extra_bytes
def scan_extra_files(index_by_cached_names, res_folder):
print "Scanning %s, checking for extra files" % res_folder
extras = 0
extra_bytes = 0
scanned = 0
for root, dirs, files in os.walk(res_folder):
for name in files:
scanned += 1
progress.write("Scanned %7.1d files (%7.1d extra)\r" %
(scanned, extras))
parent_folder = os.path.split(root)[1]
cached_name = "%s/%s" % (parent_folder, name)
if cached_name not in index_by_cached_names:
extras += 1
full_name = os.path.join(root, name)
extra_bytes += os.path.getsize(full_name)
progress.clear()
return extras, extra_bytes
def scan_missing_files(index, res_folder):
print "Scanning index, checking for missing files"
num_files = len(index)
missing = 0
missing_bytes = 0
missing_download_bytes = 0
scanned = 0
for entry in index:
scanned += 1
progress.write("Scanned %6.1d of %6.1d files (%6.1d missing)\r" % (scanned, num_files, missing))
filename = os.path.join(res_folder, entry.cached_name)
if not os.path.exists(filename):
missing += 1
missing_download_bytes += entry.compressed_size
missing_bytes += entry.size_in_bytes
continue
progress.clear()
return missing, missing_bytes, missing_download_bytes
def scan_missing_files(index, res_folder):
print "Scanning index, checking for missing files"
num_files = len(index)
missing = 0
missing_bytes = 0
missing_download_bytes = 0
scanned = 0
for entry in index:
scanned += 1
progress.write("Scanned %6.1d of %6.1d files (%6.1d missing)\r" %
(scanned, num_files, missing))
filename = os.path.join(res_folder, entry.cached_name)
if not os.path.exists(filename):
missing += 1
missing_download_bytes += entry.compressed_size
missing_bytes += entry.size_in_bytes
continue
progress.clear()
return missing, missing_bytes, missing_download_bytes
def scan_missing_files(index, res_folder):
num_files = len(index)
missing = 0
scanned = 0
missing_files = []
missing_bytes = 0
missing_bytes_on_disk = 0
for entry in index:
scanned += 1
progress.write("%6.1d of %6.1d files (%6.1d missing - %10.10s - %10.10s on disk)\r" %
(scanned, num_files, missing, format_memory(missing_bytes), format_memory(missing_bytes_on_disk)))
filename = os.path.join(res_folder, entry.cached_name)
if not os.path.exists(filename):
missing += 1
missing_files.append(entry)
missing_bytes += entry.compressed_size
missing_bytes_on_disk += entry.size_in_bytes
progress.clear()
print "%6.1d files missing - %10.10s - %10.10s on disk\r" % \
(missing, format_memory(missing_bytes), format_memory(missing_bytes_on_disk))
print
return missing_files
def main(argv):
t1=time.time()
global app
global cfg
global ignore_list
global defaultArgs
global substderr
global options
config_tool_path = os.path.split(argv[0])[0] + "/configtool.py"
fast = False
parser = optparse.OptionParser(usage="usage: %prog [options] BENCHMARK")
parser.add_option("--min", type="int", dest="min", default=1)
parser.add_option("-n", "--random", "--max", type="int", dest="n", default=-1)
parser.add_option("--offset", type="int", dest="offset", default=0)
parser.add_option("--max-sec", type="float", dest="maxsec", default=0)
parser.add_option("-d", "--debug", action="store_true", dest="debug", default=False)
parser.add_option("-f", "--fast", action="store_true", dest="fast", default=False)
parser.add_option("--threads", type="int", dest="threads", default=pbutil.cpuCount())
parser.add_option("-c", "--config", dest="config", default=None)
parser.add_option("--noisolation", action="store_true", dest="noisolation", default=False)
parser.add_option("--print", action="store_true", dest="justprint", default=False)
parser.add_option("--time", action="store_true", dest="time", default=False)
parser.add_option("--acctrials", type="int", dest="acctrials", default=None)
parser.add_option("--accimprovetries", type="int", dest="accimprovetries", default=None)
parser.add_option("--trials", type="int", dest="trials", default=None)
parser.add_option("--trials-sec", type="float", dest="trialssec", default=None)
parser.add_option("--trials-max", type="int", dest="trialsmax", default=None)
parser.add_option("--transform", dest="transform", default=None)
options,args = parser.parse_args()
if len(args) != 1:
parser.error("expected benchmark name as arg")
cfg=options.config
app=args[0]
pbutil.chdirToPetabricksRoot()
pbutil.compilePetabricks()
app = pbutil.normalizeBenchmarkName(app)
pbutil.compileBenchmarks([app])
if options.debug:
substderr = sys.__stderr__
if cfg is None:
cfg = pbutil.benchmarkToCfg(app)
defaultArgs = ['--config='+cfg, '--threads=%d'%options.threads, '--offset=%d'%options.offset, '--min=%d'%options.min]
if options.noisolation:
defaultArgs.append("--noisolation")
if options.acctrials is not None:
defaultArgs.append("--acctrials=%d"%options.acctrials)
if options.trials is not None:
defaultArgs.append("--trials=%d"%options.trials)
if options.trialssec is not None:
defaultArgs.append("--trials-sec=%f"%options.trialssec)
if options.trialsmax is not None:
defaultArgs.append("--trials-max=%d"%options.trialsmax)
if options.accimprovetries is not None:
defaultArgs.append("--accimprovetries=%d"%options.accimprovetries)
getIgnoreList()
try:
infoxml = parse(pbutil.benchmarkToInfo(app))
except:
print "Cannot parse:", pbutil.benchmarkToInfo(app)
sys.exit(-1)
#print "Reseting config entries"
#reset()
#build index of transforms
for t in infoxml.getElementsByTagName("transform"):
transforms[nameof(t)]=t
if t.getAttribute("templateChoice")=="0":
transforms[t.getAttribute("templateName")] = t
if options.transform is None:
maintx = transforms[mainname()]
else:
maintx = transforms[options.transform]
print "Call tree:"
walkCallTree(maintx, fnup=printTx)
print
print "Autotuning:"
progress.status("building work queue")
if options.n <= 0:
tasks.append(TuneTask("determineInputSizes", determineInputSizes))
if options.time:
tasks.append(TuneTask("runTimingTest", lambda:runTimingTest(maintx)))
#build list of tasks
if not options.fast:
walkCallTree(maintx, lambda tx, depth, loops: enqueueAutotuneCmds(tx, maintx, 1, depth, loops))
walkCallTree(maintx, lambda tx, depth, loops: enqueueAutotuneCmds(tx, maintx, 2, depth, loops))
if options.time:
tasks.append(TuneTask("runTimingTest", lambda:runTimingTest(maintx)))
progress.status("autotuning")
while len(tasks)>0:
w1=remainingTaskWeight()
task=tasks.pop(0)
w2=remainingTaskWeight()
progress.remaining(w1, w2)
task.run()
progress.clear()
t2=time.time()
sec=t2-t1
print "autotuning took %.2f sec"%(t2-t1)
for k,v in taskStats.items():
print " %.2f sec in %s"%(v.sec, k)
sec -= v.sec
print " %.2f sec in unknown"%sec
names=taskStats.keys()
weights=map(lambda x: x.sec/float(max(x.count, 1)), taskStats.values())
scale=len(weights)/sum(weights)
print "Suggested weights:"
print "taskStats = {" + ", ".join(map(lambda i: "'%s':TaskStats(%.2f)"%(names[i], scale*weights[i]), xrange(len(names)))) + "}"
def main(argv):
t1 = time.time()
global app
global cfg
global ignore_list
global defaultArgs
global substderr
global options
config_tool_path = os.path.split(argv[0])[0] + "/configtool.py"
fast = False
parser = optparse.OptionParser(usage="usage: %prog [options] BENCHMARK")
parser.add_option("--min", type="int", dest="min", default=1)
parser.add_option("-n",
"--random",
"--max",
type="int",
dest="n",
default=-1)
parser.add_option("--offset", type="int", dest="offset", default=0)
parser.add_option("--max-sec", type="float", dest="maxsec", default=0)
parser.add_option("-d",
"--debug",
action="store_true",
dest="debug",
default=False)
parser.add_option("-f",
"--fast",
action="store_true",
dest="fast",
default=False)
parser.add_option("--threads",
type="int",
dest="threads",
default=pbutil.cpuCount())
parser.add_option("-c", "--config", dest="config", default=None)
parser.add_option("--noisolation",
action="store_true",
dest="noisolation",
default=False)
parser.add_option("--print",
action="store_true",
dest="justprint",
default=False)
parser.add_option("--time",
action="store_true",
dest="time",
default=False)
parser.add_option("--acctrials",
type="int",
dest="acctrials",
default=None)
parser.add_option("--accimprovetries",
type="int",
dest="accimprovetries",
default=None)
parser.add_option("--trials", type="int", dest="trials", default=None)
parser.add_option("--trials-sec",
type="float",
dest="trialssec",
default=None)
parser.add_option("--trials-max",
type="int",
dest="trialsmax",
default=None)
parser.add_option("--transform", dest="transform", default=None)
options, args = parser.parse_args()
if len(args) != 1:
parser.error("expected benchmark name as arg")
cfg = options.config
app = args[0]
pbutil.chdirToPetabricksRoot()
pbutil.compilePetabricks()
app = pbutil.normalizeBenchmarkName(app)
pbutil.compileBenchmarks([app])
if options.debug:
substderr = sys.__stderr__
if cfg is None:
cfg = pbutil.benchmarkToCfg(app)
defaultArgs = [
'--config=' + cfg,
'--threads=%d' % options.threads,
'--offset=%d' % options.offset,
'--min=%d' % options.min
]
if options.noisolation:
defaultArgs.append("--noisolation")
if options.acctrials is not None:
defaultArgs.append("--acctrials=%d" % options.acctrials)
if options.trials is not None:
defaultArgs.append("--trials=%d" % options.trials)
if options.trialssec is not None:
defaultArgs.append("--trials-sec=%f" % options.trialssec)
if options.trialsmax is not None:
defaultArgs.append("--trials-max=%d" % options.trialsmax)
if options.accimprovetries is not None:
defaultArgs.append("--accimprovetries=%d" % options.accimprovetries)
getIgnoreList()
try:
infoxml = parse(pbutil.benchmarkToInfo(app))
except:
print "Cannot parse:", pbutil.benchmarkToInfo(app)
sys.exit(-1)
#print "Reseting config entries"
#reset()
#build index of transforms
for t in infoxml.getElementsByTagName("transform"):
transforms[nameof(t)] = t
if t.getAttribute("templateChoice") == "0":
transforms[t.getAttribute("templateName")] = t
if options.transform is None:
maintx = transforms[mainname()]
else:
maintx = transforms[options.transform]
print "Call tree:"
walkCallTree(maintx, fnup=printTx)
print
print "Autotuning:"
progress.status("building work queue")
if options.n <= 0:
tasks.append(TuneTask("determineInputSizes", determineInputSizes))
if options.time:
tasks.append(TuneTask("runTimingTest", lambda: runTimingTest(maintx)))
#build list of tasks
if not options.fast:
walkCallTree(
maintx, lambda tx, depth, loops: enqueueAutotuneCmds(
tx, maintx, 1, depth, loops))
walkCallTree(
maintx, lambda tx, depth, loops: enqueueAutotuneCmds(
tx, maintx, 2, depth, loops))
if options.time:
tasks.append(TuneTask("runTimingTest", lambda: runTimingTest(maintx)))
progress.status("autotuning")
while len(tasks) > 0:
w1 = remainingTaskWeight()
task = tasks.pop(0)
w2 = remainingTaskWeight()
progress.remaining(w1, w2)
task.run()
progress.clear()
t2 = time.time()
sec = t2 - t1
print "autotuning took %.2f sec" % (t2 - t1)
for k, v in taskStats.items():
print " %.2f sec in %s" % (v.sec, k)
sec -= v.sec
print " %.2f sec in unknown" % sec
names = taskStats.keys()
weights = map(lambda x: x.sec / float(max(x.count, 1)), taskStats.values())
scale = len(weights) / sum(weights)
print "Suggested weights:"
print "taskStats = {" + ", ".join(
map(lambda i: "'%s':TaskStats(%.2f)" %
(names[i], scale * weights[i]), xrange(len(names)))) + "}"