def cacheAvailable(options): """ Check to see if a cache is available, return it. """ if not os.path.exists(getStatsFileName(options.jobTree)): return None cache_file = getPreferredStatsCacheFileName(options) if not os.path.exists(cache_file): return None # check the modify times on the files, see if the cache should be recomputed mtime_stats = os.path.getmtime(getStatsFileName(options.jobTree)) mtime_cache = os.path.getmtime(cache_file) if mtime_stats > mtime_cache: # recompute cache return None # cache is fresh, return the cache return unpackData(options)
def getSettings(options): """ Collect and return the stats and config data. """ config_file = getConfigFileName(options.jobTree) stats_file = getStatsFileName(options.jobTree) try: config = ET.parse(config_file).getroot() except ET.ParseError: sys.stderr.write("The config file xml, %s, is empty.\n" % config_file) raise try: stats = ET.parse(stats_file).getroot() # Try parsing the whole file. except ET.ParseError: # If it doesn't work then we build the file incrementally sys.stderr.write("The job tree stats file is incomplete or corrupt, " "we'll try instead to parse what's in the file " "incrementally until we reach an error.\n") fH = open(stats_file, 'r') # Open the file for editing stats = ET.Element("stats") try: for event, elem in ET.iterparse(fH): if elem.tag == 'slave': stats.append(elem) except ET.ParseError: pass # Do nothing at this point finally: fH.close() return config, stats
def getPreferredStatsCacheFileName(options): """ Determine if the jobtree or the os.getcwd() version should be used. If no good option exists, return a nonexistent file path. Note you MUST check to see if the return value exists before using. """ null_file = getNullFile() location_jt = getStatsCacheFileName(options.jobTree) location_local = os.path.abspath( os.path.join(os.getcwd(), ".stats_cache.pickle")) # start by looking for the current directory cache. if os.path.exists(location_local): loc_file = open(location_local, "r") data, loc = cPickle.load(loc_file) if getStatsFileName(options.jobTree) != loc: # the local cache is from looking up a *different* jobTree location_local = null_file if os.path.exists(location_jt) and not os.path.exists(location_local): # use the jobTree directory version return location_jt elif not os.path.exists(location_jt) and os.path.exists(location_local): # use the os.getcwd() version return location_local elif os.path.exists(location_jt) and os.path.exists(location_local): # check file modify times and use the most recent version mtime_jt = os.path.getmtime(location_jt) mtime_local = os.path.getmtime(location_local) if mtime_jt > mtime_local: return location_jt else: return location_local else: return null_file
def getPreferredStatsCacheFileName(options): """ Determine if the jobtree or the os.getcwd() version should be used. If no good option exists, return a nonexistent file path. Note you MUST check to see if the return value exists before using. """ null_file = getNullFile() location_jt = getStatsCacheFileName(options.jobTree) location_local = os.path.abspath(os.path.join(os.getcwd(), ".stats_cache.pickle")) # start by looking for the current directory cache. if os.path.exists(location_local): loc_file = open(location_local, "r") data, loc = cPickle.load(loc_file) if getStatsFileName(options.jobTree) != loc: # the local cache is from looking up a *different* jobTree location_local = null_file if os.path.exists(location_jt) and not os.path.exists(location_local): # use the jobTree directory version return location_jt elif not os.path.exists(location_jt) and os.path.exists(location_local): # use the os.getcwd() version return location_local elif os.path.exists(location_jt) and os.path.exists(location_local): # check file modify times and use the most recent version mtime_jt = os.path.getmtime(location_jt) mtime_local = os.path.getmtime(location_local) if mtime_jt > mtime_local: return location_jt else: return location_local else: return null_file
def checkOptions(options, args, parser): """ Check options, throw parser.error() if something goes wrong """ logger.info("Parsed arguments") if len(sys.argv) == 1: parser.print_help() sys.exit(0) assert len(args) <= 1 # Only jobtree may be specified as argument if len(args) == 1: # Allow jobTree directory as arg options.jobTree = args[0] logger.info("Checking if we have files for job tree") if options.jobTree == None: parser.error("Specify --jobTree") if not os.path.exists(options.jobTree): parser.error("--jobTree %s does not exist" % options.jobTree) if not os.path.isdir(options.jobTree): parser.error("--jobTree %s is not a directory" % options.jobTree) if not os.path.isfile(getConfigFileName(options.jobTree)): parser.error("A valid job tree must contain the config file") if not os.path.isfile(getStatsFileName(options.jobTree)): parser.error("The job-tree was run without the --stats flag, " "so no stats were created") defaultCategories = ["time", "clock", "wait", "memory"] if options.categories is None: options.categories = defaultCategories else: options.categories = map(lambda x: x.lower(), options.categories.split(",")) for c in options.categories: if c not in defaultCategories: parser.error("Unknown category %s. Must be from %s" % (c, str(defaultCategories))) extraSort = ["count", "alpha"] if options.sortCategory is not None: if (options.sortCategory not in defaultCategories and options.sortCategory not in extraSort): parser.error("Unknown --sortCategory %s. Must be from %s" % (options.sortCategory, str(defaultCategories + extraSort))) sortFields = ["min", "med", "ave", "max", "total"] if options.sortField is not None: if (options.sortField not in sortFields): parser.error("Unknown --sortField %s. Must be from %s" % (options.sortField, str(sortFields))) logger.info("Checked arguments")
def checkOptions(options, args, parser): """ Check options, throw parser.error() if something goes wrong """ logger.info("Parsed arguments") if len(sys.argv) == 1: parser.print_help() sys.exit(0) assert len(args) <= 1 # Only jobtree may be specified as argument if len(args) == 1: # Allow jobTree directory as arg options.jobTree = args[0] logger.info("Checking if we have files for job tree") if options.jobTree == None: parser.error("Specify --jobTree") if not os.path.exists(options.jobTree): parser.error("--jobTree %s does not exist" % options.jobTree) if not os.path.isdir(options.jobTree): parser.error("--jobTree %s is not a directory" % options.jobTree) if not os.path.isfile(getConfigFileName(options.jobTree)): parser.error("A valid job tree must contain the config file") if not os.path.isfile(getStatsFileName(options.jobTree)): parser.error("The job-tree was run without the --stats flag, " "so no stats were created") defaultCategories = ["time", "clock", "wait", "memory"] if options.categories is None: options.categories = defaultCategories else: options.categories = map(lambda x: x.lower(), options.categories.split(",")) for c in options.categories: if c not in defaultCategories: parser.error("Unknown category %s. Must be from %s" % (c, str(defaultCategories))) extraSort = ["count", "alpha"] if options.sortCategory is not None: if (options.sortCategory not in defaultCategories and options.sortCategory not in extraSort): parser.error( "Unknown --sortCategory %s. Must be from %s" % (options.sortCategory, str(defaultCategories + extraSort))) sortFields = ["min", "med", "ave", "max", "total"] if options.sortField is not None: if (options.sortField not in sortFields): parser.error("Unknown --sortField %s. Must be from %s" % (options.sortField, str(sortFields))) logger.info("Checked arguments")
def unpackData(options): """unpackData() opens up the pickle of the last run and pulls out all the relevant data. """ cache_file = getPreferredStatsCacheFileName(options) if not os.path.exists(cache_file): return None if os.path.exists(cache_file): f = open(cache_file, "r") try: data, location = cPickle.load(f) except EOFError: # bad cache. return None finally: f.close() if location == getStatsFileName(options.jobTree): return data return None
def packData(data, options): """ packData stores all of the data in the appropriate pickle cache file. """ stats_file = getStatsFileName(options.jobTree) cache_file = getStatsCacheFileName(options.jobTree) try: # try to write to the jobTree directory payload = (data, stats_file) f = open(cache_file, "wb") cPickle.dump(payload, f, 2) # 2 is binary format f.close() except IOError: if not options.cache: return # try to write to the current working directory only if --cache cache_file = os.path.abspath( os.path.join(os.getcwd(), ".stats_cache.pickle")) payload = (data, stats_file) f = open(cache_file, "wb") cPickle.dump(payload, f, 2) # 2 is binary format f.close()
def packData(data, options): """ packData stores all of the data in the appropriate pickle cache file. """ stats_file = getStatsFileName(options.jobTree) cache_file = getStatsCacheFileName(options.jobTree) try: # try to write to the jobTree directory payload = (data, stats_file) f = open(cache_file, "wb") cPickle.dump(payload, f, 2) # 2 is binary format f.close() except IOError: if not options.cache: return # try to write to the current working directory only if --cache cache_file = os.path.abspath(os.path.join(os.getcwd(), ".stats_cache.pickle")) payload = (data, stats_file) f = open(cache_file, "wb") cPickle.dump(payload, f, 2) # 2 is binary format f.close()