Esempio n. 1
0
def delete_extra_files(index_by_cached_names, res_folder):
    extras = 0
    scanned = 0
    extra_files = []
    for root, dirs, files in os.walk(res_folder):
        for name in files:
            scanned += 1
            progress.write("Scanned %7.1d files (%7.1d extra)\r" % (scanned, extras))
            parent_folder = os.path.split(root)[1]
            cached_name = "%s/%s" % (parent_folder, name)
            if cached_name not in index_by_cached_names:
                extras += 1
                extra_files.append(os.path.join(root, name))
    progress.clear()

    removed = 0
    for filename in extra_files:
        try:
            progress.write("Removed %7.1d of %7.1d files\r" % (removed, extras))
            os.remove(filename)
            removed += 1
        except IOError:
            pass

    return extras
 def resetDeviceState(self):
     if self.deviceState.has_key(self.device):
         del self.deviceState[self.device]
         pushConfig.recycleConfig(self.config)
     progress.clear(self.taskId, self.device)
     if allTaskProgress.has_key(self.taskId) and allTaskProgress[self.taskId] == self.device:
         del allTaskProgress[self.taskId]
Esempio n. 3
0
def scan_missing_files(index, res_folder):
    num_files = len(index)
    missing = 0
    scanned = 0
    missing_files = []
    missing_bytes = 0
    missing_bytes_on_disk = 0
    for entry in index:
        scanned += 1
        progress.write(
            "%6.1d of %6.1d files (%6.1d missing - %10.10s - %10.10s on disk)\r"
            % (scanned, num_files, missing, format_memory(missing_bytes),
               format_memory(missing_bytes_on_disk)))
        filename = os.path.join(res_folder, entry.cached_name)
        if not os.path.exists(filename):
            missing += 1
            missing_files.append(entry)
            missing_bytes += entry.compressed_size
            missing_bytes_on_disk += entry.size_in_bytes

    progress.clear()
    print "%6.1d files missing - %10.10s - %10.10s on disk\r" % \
          (missing, format_memory(missing_bytes), format_memory(missing_bytes_on_disk))
    print

    return missing_files
Esempio n. 4
0
def verify_cache(index, res_folder):
    num_files = len(index)

    missing = 0
    corrupt = 0
    scanned = 0
    for entry in index:
        scanned += 1
        progress.write("Scanned %6.1d of %6.1d files (%6.1d corrupt, %6.1d missing)\r" %
                       (scanned, num_files, corrupt, missing))
        filename = os.path.join(res_folder, entry.cached_name)
        if not os.path.exists(filename):
            missing += 1
            continue

        checksum = calc_checksum(filename)
        if checksum != entry.md5_checksum:
            corrupt += 1
            try:
                os.remove(filename)
            except IOError:
                pass

    progress.clear()
    print "Verified %d files:" % num_files
    print "  %6.1d files corrupt" % corrupt
    print "  %6.1d files not yet downloaded" % missing

    return corrupt, missing
Esempio n. 5
0
def main():
    progress.stream = sys.stdout

    if len(sys.argv) < 2:
        run_interactive()
        sys.exit()

    parser = argparse.ArgumentParser(
        description="rescache is a tool for verifying and managing the EVE shared resource cache."
    )
    parser.add_argument(
        "-i", "--index",
        default=DEFAULT_INDEX_FILENAME,
        help="The name of an index file to use - defaults to %s" % DEFAULT_INDEX_FILENAME
    )
    parser.add_argument(
        "-c", "--cache",
        default=get_shared_cache_folder(),
        help="The location of the shared cache to use - defaults to what the EVE client uses"
    )
    parser.add_argument(
        "-d", "--dir",
        action='append',
        help="Additional directories (EVE installs) to read indexes from"
    )
    subparsers = parser.add_subparsers()

    parser_verify = subparsers.add_parser("verify")
    parser_verify.set_defaults(func=verify_command)

    parser_diff = subparsers.add_parser("diff")
    parser_diff.set_defaults(func=diff_command)

    parser_purge = subparsers.add_parser("purge")
    parser_purge.set_defaults(func=purge_command)

    parser_download = subparsers.add_parser("download")
    parser_download.set_defaults(func=download_command)

    parser_move = subparsers.add_parser("move")
    parser_move.add_argument(
        "destination",
        help="The name of the cache folder. This folder must not exist already."
    )
    parser_move.set_defaults(func=move_command)

    args = parser.parse_args()

    try:
        args.func(args)

    except KeyboardInterrupt:
        progress.clear()
        print "Operation cancelled"
        sys.exit(1)
Esempio n. 6
0
def main():
    progress.stream = sys.stdout

    if len(sys.argv) < 2:
        run_interactive()
        sys.exit()

    parser = argparse.ArgumentParser(
        description=
        "rescache is a tool for verifying and managing the EVE shared resource cache."
    )
    parser.add_argument(
        "-i",
        "--index",
        default=DEFAULT_INDEX_FILENAME,
        help="The name of an index file to use - defaults to %s" %
        DEFAULT_INDEX_FILENAME)
    parser.add_argument(
        "-c",
        "--cache",
        default=get_shared_cache_folder(),
        help=
        "The location of the shared cache to use - defaults to what the EVE client uses"
    )
    subparsers = parser.add_subparsers()

    parser_verify = subparsers.add_parser("verify")
    parser_verify.set_defaults(func=verify_command)

    parser_diff = subparsers.add_parser("diff")
    parser_diff.set_defaults(func=diff_command)

    parser_purge = subparsers.add_parser("purge")
    parser_purge.set_defaults(func=purge_command)

    parser_download = subparsers.add_parser("download")
    parser_download.set_defaults(func=download_command)

    parser_move = subparsers.add_parser("move")
    parser_move.add_argument(
        "destination",
        help="The name of the cache folder. This folder must not exist already."
    )
    parser_move.set_defaults(func=move_command)

    args = parser.parse_args()

    try:
        args.func(args)

    except KeyboardInterrupt:
        progress.clear()
        print "Operation cancelled"
        sys.exit(1)
Esempio n. 7
0
def download_missing_files(res_folder, files_to_download):
    q = Queue.Queue()
    for f in files_to_download:
        q.put(f)

    downloaded_files = 0
    old_size = q.qsize()
    num_files = len(files_to_download)

    thread_list = []

    for i in range(DOWNLOAD_THREAD_COUNT):
        t = DownloadThread(res_folder, q)
        thread_list.append(t)
        t.start()

    try:
        while not q.empty():
            progress.write("Downloaded %6.1d of %6.1d files\r" %
                           (downloaded_files, num_files))
            new_size = q.qsize()
            d_size = old_size - new_size
            if d_size:
                downloaded_files += d_size
            old_size = new_size
            time.sleep(0.5)

        progress.clear()

        num_failed = 0
        num_succeeded = 0
        for t in thread_list:
            t.join()
            num_failed += t.failed
            num_succeeded += t.succeeded
            if t.messages:
                print t.messages

        return num_succeeded, num_failed

    except KeyboardInterrupt:
        progress.clear()
        print "Stopping download threads"
        for t in thread_list:
            t.stop()
        for t in thread_list:
            t.join()
        raise
Esempio n. 8
0
def download_missing_files(res_folder, files_to_download):
    q = Queue.Queue()
    for f in files_to_download:
        q.put(f)

    downloaded_files = 0
    old_size = q.qsize()
    num_files = len(files_to_download)

    thread_list = []

    for i in range(DOWNLOAD_THREAD_COUNT):
        t = DownloadThread(res_folder, q)
        thread_list.append(t)
        t.start()

    try:
        while not q.empty():
            progress.write("Downloaded %6.1d of %6.1d files\r" % (downloaded_files, num_files))
            new_size = q.qsize()
            d_size = old_size - new_size
            if d_size:
                downloaded_files += d_size
            old_size = new_size
            time.sleep(0.5)

        progress.clear()

        num_failed = 0
        num_succeeded = 0
        for t in thread_list:
            t.join()
            num_failed += t.failed
            num_succeeded += t.succeeded
            if t.messages:
                print t.messages

        return num_succeeded, num_failed

    except KeyboardInterrupt:
        progress.clear()
        print "Stopping download threads"
        for t in thread_list:
            t.stop()
        for t in thread_list:
            t.join()
        raise
Esempio n. 9
0
def scan_extra_files(index_by_cached_names, res_folder):
    print "Scanning %s, checking for extra files" % res_folder
    extras = 0
    extra_bytes = 0
    scanned = 0
    for root, dirs, files in os.walk(res_folder):
        for name in files:
            scanned += 1
            progress.write("Scanned %7.1d files (%7.1d extra)\r" % (scanned, extras))
            parent_folder = os.path.split(root)[1]
            cached_name = "%s/%s" % (parent_folder, name)
            if cached_name not in index_by_cached_names:
                extras += 1
                full_name = os.path.join(root, name)
                extra_bytes += os.path.getsize(full_name)
    progress.clear()
    return extras, extra_bytes
Esempio n. 10
0
def scan_extra_files(index_by_cached_names, res_folder):
    print "Scanning %s, checking for extra files" % res_folder
    extras = 0
    extra_bytes = 0
    scanned = 0
    for root, dirs, files in os.walk(res_folder):
        for name in files:
            scanned += 1
            progress.write("Scanned %7.1d files (%7.1d extra)\r" %
                           (scanned, extras))
            parent_folder = os.path.split(root)[1]
            cached_name = "%s/%s" % (parent_folder, name)
            if cached_name not in index_by_cached_names:
                extras += 1
                full_name = os.path.join(root, name)
                extra_bytes += os.path.getsize(full_name)
    progress.clear()
    return extras, extra_bytes
Esempio n. 11
0
def scan_missing_files(index, res_folder):
    print "Scanning index, checking for missing files"
    num_files = len(index)
    missing = 0
    missing_bytes = 0
    missing_download_bytes = 0
    scanned = 0
    for entry in index:
        scanned += 1
        progress.write("Scanned %6.1d of %6.1d files (%6.1d missing)\r" % (scanned, num_files, missing))
        filename = os.path.join(res_folder, entry.cached_name)
        if not os.path.exists(filename):
            missing += 1
            missing_download_bytes += entry.compressed_size
            missing_bytes += entry.size_in_bytes
            continue
    progress.clear()
    return missing, missing_bytes, missing_download_bytes
Esempio n. 12
0
def scan_missing_files(index, res_folder):
    print "Scanning index, checking for missing files"
    num_files = len(index)
    missing = 0
    missing_bytes = 0
    missing_download_bytes = 0
    scanned = 0
    for entry in index:
        scanned += 1
        progress.write("Scanned %6.1d of %6.1d files (%6.1d missing)\r" %
                       (scanned, num_files, missing))
        filename = os.path.join(res_folder, entry.cached_name)
        if not os.path.exists(filename):
            missing += 1
            missing_download_bytes += entry.compressed_size
            missing_bytes += entry.size_in_bytes
            continue
    progress.clear()
    return missing, missing_bytes, missing_download_bytes
Esempio n. 13
0
def scan_missing_files(index, res_folder):
    num_files = len(index)
    missing = 0
    scanned = 0
    missing_files = []
    missing_bytes = 0
    missing_bytes_on_disk = 0
    for entry in index:
        scanned += 1
        progress.write("%6.1d of %6.1d files (%6.1d missing - %10.10s - %10.10s on disk)\r" %
                       (scanned, num_files, missing, format_memory(missing_bytes), format_memory(missing_bytes_on_disk)))
        filename = os.path.join(res_folder, entry.cached_name)
        if not os.path.exists(filename):
            missing += 1
            missing_files.append(entry)
            missing_bytes += entry.compressed_size
            missing_bytes_on_disk += entry.size_in_bytes

    progress.clear()
    print "%6.1d files missing - %10.10s - %10.10s on disk\r" % \
          (missing, format_memory(missing_bytes), format_memory(missing_bytes_on_disk))
    print

    return missing_files
Esempio n. 14
0
def main(argv):
  t1=time.time()

  global app
  global cfg 
  global ignore_list
  global defaultArgs
  global substderr
  global options 

  config_tool_path = os.path.split(argv[0])[0] + "/configtool.py"
  fast = False

  parser = optparse.OptionParser(usage="usage: %prog [options] BENCHMARK")
  parser.add_option("--min", type="int", dest="min", default=1)
  parser.add_option("-n", "--random", "--max", type="int", dest="n", default=-1)
  parser.add_option("--offset", type="int", dest="offset", default=0)
  parser.add_option("--max-sec", type="float", dest="maxsec", default=0)
  parser.add_option("-d", "--debug",  action="store_true", dest="debug", default=False)
  parser.add_option("-f", "--fast",  action="store_true", dest="fast", default=False)
  parser.add_option("--threads",      type="int", dest="threads", default=pbutil.cpuCount())
  parser.add_option("-c", "--config", dest="config", default=None)
  parser.add_option("--noisolation", action="store_true", dest="noisolation", default=False)
  parser.add_option("--print", action="store_true", dest="justprint", default=False)
  parser.add_option("--time", action="store_true", dest="time", default=False)
  parser.add_option("--acctrials", type="int", dest="acctrials", default=None)
  parser.add_option("--accimprovetries", type="int", dest="accimprovetries", default=None)
  parser.add_option("--trials", type="int", dest="trials", default=None)
  parser.add_option("--trials-sec", type="float", dest="trialssec", default=None)
  parser.add_option("--trials-max", type="int", dest="trialsmax", default=None)
  parser.add_option("--transform", dest="transform", default=None)
  options,args = parser.parse_args()

  if len(args) != 1:
    parser.error("expected benchmark name as arg")

  cfg=options.config
  app=args[0]

  pbutil.chdirToPetabricksRoot()
  pbutil.compilePetabricks()
  app = pbutil.normalizeBenchmarkName(app)
  pbutil.compileBenchmarks([app])
  
  if options.debug:
    substderr = sys.__stderr__

  if cfg is None:
    cfg = pbutil.benchmarkToCfg(app)

  defaultArgs = ['--config='+cfg, '--threads=%d'%options.threads, '--offset=%d'%options.offset, '--min=%d'%options.min]

  if options.noisolation:
    defaultArgs.append("--noisolation")

  if options.acctrials is not None:
    defaultArgs.append("--acctrials=%d"%options.acctrials)
  if options.trials is not None:
    defaultArgs.append("--trials=%d"%options.trials)
  if options.trialssec is not None:
    defaultArgs.append("--trials-sec=%f"%options.trialssec)
  if options.trialsmax is not None:
    defaultArgs.append("--trials-max=%d"%options.trialsmax)
  if options.accimprovetries is not None:
    defaultArgs.append("--accimprovetries=%d"%options.accimprovetries)

  getIgnoreList()

  try:
    infoxml = parse(pbutil.benchmarkToInfo(app))
  except:
    print "Cannot parse:", pbutil.benchmarkToInfo(app)
    sys.exit(-1)

 #print "Reseting config entries"
 #reset()

  #build index of transforms
  for t in infoxml.getElementsByTagName("transform"):
    transforms[nameof(t)]=t
    if t.getAttribute("templateChoice")=="0":
      transforms[t.getAttribute("templateName")] = t

  if options.transform is None:
    maintx = transforms[mainname()]
  else:
    maintx = transforms[options.transform]
  
  print "Call tree:"
  walkCallTree(maintx, fnup=printTx)
  print
  print "Autotuning:"

  progress.status("building work queue")
 
  if options.n <= 0:
    tasks.append(TuneTask("determineInputSizes", determineInputSizes))
    
  if options.time:
    tasks.append(TuneTask("runTimingTest", lambda:runTimingTest(maintx)))

  #build list of tasks
  if not options.fast:
    walkCallTree(maintx, lambda tx, depth, loops: enqueueAutotuneCmds(tx, maintx, 1, depth, loops))
  walkCallTree(maintx, lambda tx, depth, loops: enqueueAutotuneCmds(tx, maintx, 2, depth, loops))
  
  if options.time:
    tasks.append(TuneTask("runTimingTest", lambda:runTimingTest(maintx)))

  progress.status("autotuning")

  while len(tasks)>0:
    w1=remainingTaskWeight()
    task=tasks.pop(0)
    w2=remainingTaskWeight()
    progress.remaining(w1, w2)
    task.run()
  progress.clear()

  t2=time.time()
  sec=t2-t1

  

  print "autotuning took %.2f sec"%(t2-t1)
  for k,v in taskStats.items():
    print "  %.2f sec in %s"%(v.sec, k)
    sec -= v.sec
  print "  %.2f sec in unknown"%sec
  
  names=taskStats.keys()
  weights=map(lambda x: x.sec/float(max(x.count, 1)), taskStats.values())
  scale=len(weights)/sum(weights)
  print "Suggested weights:"
  print "taskStats = {" + ", ".join(map(lambda i: "'%s':TaskStats(%.2f)"%(names[i], scale*weights[i]), xrange(len(names)))) + "}"
Esempio n. 15
0
def main(argv):
    t1 = time.time()

    global app
    global cfg
    global ignore_list
    global defaultArgs
    global substderr
    global options

    config_tool_path = os.path.split(argv[0])[0] + "/configtool.py"
    fast = False

    parser = optparse.OptionParser(usage="usage: %prog [options] BENCHMARK")
    parser.add_option("--min", type="int", dest="min", default=1)
    parser.add_option("-n",
                      "--random",
                      "--max",
                      type="int",
                      dest="n",
                      default=-1)
    parser.add_option("--offset", type="int", dest="offset", default=0)
    parser.add_option("--max-sec", type="float", dest="maxsec", default=0)
    parser.add_option("-d",
                      "--debug",
                      action="store_true",
                      dest="debug",
                      default=False)
    parser.add_option("-f",
                      "--fast",
                      action="store_true",
                      dest="fast",
                      default=False)
    parser.add_option("--threads",
                      type="int",
                      dest="threads",
                      default=pbutil.cpuCount())
    parser.add_option("-c", "--config", dest="config", default=None)
    parser.add_option("--noisolation",
                      action="store_true",
                      dest="noisolation",
                      default=False)
    parser.add_option("--print",
                      action="store_true",
                      dest="justprint",
                      default=False)
    parser.add_option("--time",
                      action="store_true",
                      dest="time",
                      default=False)
    parser.add_option("--acctrials",
                      type="int",
                      dest="acctrials",
                      default=None)
    parser.add_option("--accimprovetries",
                      type="int",
                      dest="accimprovetries",
                      default=None)
    parser.add_option("--trials", type="int", dest="trials", default=None)
    parser.add_option("--trials-sec",
                      type="float",
                      dest="trialssec",
                      default=None)
    parser.add_option("--trials-max",
                      type="int",
                      dest="trialsmax",
                      default=None)
    parser.add_option("--transform", dest="transform", default=None)
    options, args = parser.parse_args()

    if len(args) != 1:
        parser.error("expected benchmark name as arg")

    cfg = options.config
    app = args[0]

    pbutil.chdirToPetabricksRoot()
    pbutil.compilePetabricks()
    app = pbutil.normalizeBenchmarkName(app)
    pbutil.compileBenchmarks([app])

    if options.debug:
        substderr = sys.__stderr__

    if cfg is None:
        cfg = pbutil.benchmarkToCfg(app)

    defaultArgs = [
        '--config=' + cfg,
        '--threads=%d' % options.threads,
        '--offset=%d' % options.offset,
        '--min=%d' % options.min
    ]

    if options.noisolation:
        defaultArgs.append("--noisolation")

    if options.acctrials is not None:
        defaultArgs.append("--acctrials=%d" % options.acctrials)
    if options.trials is not None:
        defaultArgs.append("--trials=%d" % options.trials)
    if options.trialssec is not None:
        defaultArgs.append("--trials-sec=%f" % options.trialssec)
    if options.trialsmax is not None:
        defaultArgs.append("--trials-max=%d" % options.trialsmax)
    if options.accimprovetries is not None:
        defaultArgs.append("--accimprovetries=%d" % options.accimprovetries)

    getIgnoreList()

    try:
        infoxml = parse(pbutil.benchmarkToInfo(app))
    except:
        print "Cannot parse:", pbutil.benchmarkToInfo(app)
        sys.exit(-1)

#print "Reseting config entries"
#reset()

#build index of transforms
    for t in infoxml.getElementsByTagName("transform"):
        transforms[nameof(t)] = t
        if t.getAttribute("templateChoice") == "0":
            transforms[t.getAttribute("templateName")] = t

    if options.transform is None:
        maintx = transforms[mainname()]
    else:
        maintx = transforms[options.transform]

    print "Call tree:"
    walkCallTree(maintx, fnup=printTx)
    print
    print "Autotuning:"

    progress.status("building work queue")

    if options.n <= 0:
        tasks.append(TuneTask("determineInputSizes", determineInputSizes))

    if options.time:
        tasks.append(TuneTask("runTimingTest", lambda: runTimingTest(maintx)))

    #build list of tasks
    if not options.fast:
        walkCallTree(
            maintx, lambda tx, depth, loops: enqueueAutotuneCmds(
                tx, maintx, 1, depth, loops))
    walkCallTree(
        maintx, lambda tx, depth, loops: enqueueAutotuneCmds(
            tx, maintx, 2, depth, loops))

    if options.time:
        tasks.append(TuneTask("runTimingTest", lambda: runTimingTest(maintx)))

    progress.status("autotuning")

    while len(tasks) > 0:
        w1 = remainingTaskWeight()
        task = tasks.pop(0)
        w2 = remainingTaskWeight()
        progress.remaining(w1, w2)
        task.run()
    progress.clear()

    t2 = time.time()
    sec = t2 - t1

    print "autotuning took %.2f sec" % (t2 - t1)
    for k, v in taskStats.items():
        print "  %.2f sec in %s" % (v.sec, k)
        sec -= v.sec
    print "  %.2f sec in unknown" % sec

    names = taskStats.keys()
    weights = map(lambda x: x.sec / float(max(x.count, 1)), taskStats.values())
    scale = len(weights) / sum(weights)
    print "Suggested weights:"
    print "taskStats = {" + ", ".join(
        map(lambda i: "'%s':TaskStats(%.2f)" %
            (names[i], scale * weights[i]), xrange(len(names)))) + "}"