def cacheAvailable(options):
""" Check to see if a cache is available, return it.
"""
if not os.path.exists(getStatsFileName(options.jobTree)):
return None
cache_file = getPreferredStatsCacheFileName(options)
if not os.path.exists(cache_file):
return None
# check the modify times on the files, see if the cache should be recomputed
mtime_stats = os.path.getmtime(getStatsFileName(options.jobTree))
mtime_cache = os.path.getmtime(cache_file)
if mtime_stats > mtime_cache:
# recompute cache
return None
# cache is fresh, return the cache
return unpackData(options)
def getSettings(options):
""" Collect and return the stats and config data.
"""
config_file = getConfigFileName(options.jobTree)
stats_file = getStatsFileName(options.jobTree)
try:
config = ET.parse(config_file).getroot()
except ET.ParseError:
sys.stderr.write("The config file xml, %s, is empty.\n" % config_file)
raise
try:
stats = ET.parse(stats_file).getroot() # Try parsing the whole file.
except ET.ParseError: # If it doesn't work then we build the file incrementally
sys.stderr.write("The job tree stats file is incomplete or corrupt, "
"we'll try instead to parse what's in the file "
"incrementally until we reach an error.\n")
fH = open(stats_file, 'r') # Open the file for editing
stats = ET.Element("stats")
try:
for event, elem in ET.iterparse(fH):
if elem.tag == 'slave':
stats.append(elem)
except ET.ParseError:
pass # Do nothing at this point
finally:
fH.close()
return config, stats
def getPreferredStatsCacheFileName(options):
""" Determine if the jobtree or the os.getcwd() version should be used.
If no good option exists, return a nonexistent file path.
Note you MUST check to see if the return value exists before using.
"""
null_file = getNullFile()
location_jt = getStatsCacheFileName(options.jobTree)
location_local = os.path.abspath(
os.path.join(os.getcwd(), ".stats_cache.pickle"))
# start by looking for the current directory cache.
if os.path.exists(location_local):
loc_file = open(location_local, "r")
data, loc = cPickle.load(loc_file)
if getStatsFileName(options.jobTree) != loc:
# the local cache is from looking up a *different* jobTree
location_local = null_file
if os.path.exists(location_jt) and not os.path.exists(location_local):
# use the jobTree directory version
return location_jt
elif not os.path.exists(location_jt) and os.path.exists(location_local):
# use the os.getcwd() version
return location_local
elif os.path.exists(location_jt) and os.path.exists(location_local):
# check file modify times and use the most recent version
mtime_jt = os.path.getmtime(location_jt)
mtime_local = os.path.getmtime(location_local)
if mtime_jt > mtime_local:
return location_jt
else:
return location_local
else:
return null_file
def cacheAvailable(options):
""" Check to see if a cache is available, return it.
"""
if not os.path.exists(getStatsFileName(options.jobTree)):
return None
cache_file = getPreferredStatsCacheFileName(options)
if not os.path.exists(cache_file):
return None
# check the modify times on the files, see if the cache should be recomputed
mtime_stats = os.path.getmtime(getStatsFileName(options.jobTree))
mtime_cache = os.path.getmtime(cache_file)
if mtime_stats > mtime_cache:
# recompute cache
return None
# cache is fresh, return the cache
return unpackData(options)
def getPreferredStatsCacheFileName(options):
""" Determine if the jobtree or the os.getcwd() version should be used.
If no good option exists, return a nonexistent file path.
Note you MUST check to see if the return value exists before using.
"""
null_file = getNullFile()
location_jt = getStatsCacheFileName(options.jobTree)
location_local = os.path.abspath(os.path.join(os.getcwd(),
".stats_cache.pickle"))
# start by looking for the current directory cache.
if os.path.exists(location_local):
loc_file = open(location_local, "r")
data, loc = cPickle.load(loc_file)
if getStatsFileName(options.jobTree) != loc:
# the local cache is from looking up a *different* jobTree
location_local = null_file
if os.path.exists(location_jt) and not os.path.exists(location_local):
# use the jobTree directory version
return location_jt
elif not os.path.exists(location_jt) and os.path.exists(location_local):
# use the os.getcwd() version
return location_local
elif os.path.exists(location_jt) and os.path.exists(location_local):
# check file modify times and use the most recent version
mtime_jt = os.path.getmtime(location_jt)
mtime_local = os.path.getmtime(location_local)
if mtime_jt > mtime_local:
return location_jt
else:
return location_local
else:
return null_file
def getSettings(options):
""" Collect and return the stats and config data.
"""
config_file = getConfigFileName(options.jobTree)
stats_file = getStatsFileName(options.jobTree)
try:
config = ET.parse(config_file).getroot()
except ET.ParseError:
sys.stderr.write("The config file xml, %s, is empty.\n" % config_file)
raise
try:
stats = ET.parse(stats_file).getroot() # Try parsing the whole file.
except ET.ParseError: # If it doesn't work then we build the file incrementally
sys.stderr.write("The job tree stats file is incomplete or corrupt, "
"we'll try instead to parse what's in the file "
"incrementally until we reach an error.\n")
fH = open(stats_file, 'r') # Open the file for editing
stats = ET.Element("stats")
try:
for event, elem in ET.iterparse(fH):
if elem.tag == 'slave':
stats.append(elem)
except ET.ParseError:
pass # Do nothing at this point
finally:
fH.close()
return config, stats
def checkOptions(options, args, parser):
""" Check options, throw parser.error() if something goes wrong
"""
logger.info("Parsed arguments")
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
assert len(args) <= 1 # Only jobtree may be specified as argument
if len(args) == 1: # Allow jobTree directory as arg
options.jobTree = args[0]
logger.info("Checking if we have files for job tree")
if options.jobTree == None:
parser.error("Specify --jobTree")
if not os.path.exists(options.jobTree):
parser.error("--jobTree %s does not exist"
% options.jobTree)
if not os.path.isdir(options.jobTree):
parser.error("--jobTree %s is not a directory"
% options.jobTree)
if not os.path.isfile(getConfigFileName(options.jobTree)):
parser.error("A valid job tree must contain the config file")
if not os.path.isfile(getStatsFileName(options.jobTree)):
parser.error("The job-tree was run without the --stats flag, "
"so no stats were created")
defaultCategories = ["time", "clock", "wait", "memory"]
if options.categories is None:
options.categories = defaultCategories
else:
options.categories = map(lambda x: x.lower(),
options.categories.split(","))
for c in options.categories:
if c not in defaultCategories:
parser.error("Unknown category %s. Must be from %s"
% (c, str(defaultCategories)))
extraSort = ["count", "alpha"]
if options.sortCategory is not None:
if (options.sortCategory not in defaultCategories and
options.sortCategory not in extraSort):
parser.error("Unknown --sortCategory %s. Must be from %s"
% (options.sortCategory,
str(defaultCategories + extraSort)))
sortFields = ["min", "med", "ave", "max", "total"]
if options.sortField is not None:
if (options.sortField not in sortFields):
parser.error("Unknown --sortField %s. Must be from %s"
% (options.sortField, str(sortFields)))
logger.info("Checked arguments")
def checkOptions(options, args, parser):
""" Check options, throw parser.error() if something goes wrong
"""
logger.info("Parsed arguments")
if len(sys.argv) == 1:
parser.print_help()
sys.exit(0)
assert len(args) <= 1 # Only jobtree may be specified as argument
if len(args) == 1: # Allow jobTree directory as arg
options.jobTree = args[0]
logger.info("Checking if we have files for job tree")
if options.jobTree == None:
parser.error("Specify --jobTree")
if not os.path.exists(options.jobTree):
parser.error("--jobTree %s does not exist" % options.jobTree)
if not os.path.isdir(options.jobTree):
parser.error("--jobTree %s is not a directory" % options.jobTree)
if not os.path.isfile(getConfigFileName(options.jobTree)):
parser.error("A valid job tree must contain the config file")
if not os.path.isfile(getStatsFileName(options.jobTree)):
parser.error("The job-tree was run without the --stats flag, "
"so no stats were created")
defaultCategories = ["time", "clock", "wait", "memory"]
if options.categories is None:
options.categories = defaultCategories
else:
options.categories = map(lambda x: x.lower(),
options.categories.split(","))
for c in options.categories:
if c not in defaultCategories:
parser.error("Unknown category %s. Must be from %s" %
(c, str(defaultCategories)))
extraSort = ["count", "alpha"]
if options.sortCategory is not None:
if (options.sortCategory not in defaultCategories
and options.sortCategory not in extraSort):
parser.error(
"Unknown --sortCategory %s. Must be from %s" %
(options.sortCategory, str(defaultCategories + extraSort)))
sortFields = ["min", "med", "ave", "max", "total"]
if options.sortField is not None:
if (options.sortField not in sortFields):
parser.error("Unknown --sortField %s. Must be from %s" %
(options.sortField, str(sortFields)))
logger.info("Checked arguments")
def unpackData(options):
"""unpackData() opens up the pickle of the last run and pulls out
all the relevant data.
"""
cache_file = getPreferredStatsCacheFileName(options)
if not os.path.exists(cache_file):
return None
if os.path.exists(cache_file):
f = open(cache_file, "r")
try:
data, location = cPickle.load(f)
except EOFError:
# bad cache.
return None
finally:
f.close()
if location == getStatsFileName(options.jobTree):
return data
return None
def unpackData(options):
"""unpackData() opens up the pickle of the last run and pulls out
all the relevant data.
"""
cache_file = getPreferredStatsCacheFileName(options)
if not os.path.exists(cache_file):
return None
if os.path.exists(cache_file):
f = open(cache_file, "r")
try:
data, location = cPickle.load(f)
except EOFError:
# bad cache.
return None
finally:
f.close()
if location == getStatsFileName(options.jobTree):
return data
return None
def packData(data, options):
""" packData stores all of the data in the appropriate pickle cache file.
"""
stats_file = getStatsFileName(options.jobTree)
cache_file = getStatsCacheFileName(options.jobTree)
try:
# try to write to the jobTree directory
payload = (data, stats_file)
f = open(cache_file, "wb")
cPickle.dump(payload, f, 2) # 2 is binary format
f.close()
except IOError:
if not options.cache:
return
# try to write to the current working directory only if --cache
cache_file = os.path.abspath(
os.path.join(os.getcwd(), ".stats_cache.pickle"))
payload = (data, stats_file)
f = open(cache_file, "wb")
cPickle.dump(payload, f, 2) # 2 is binary format
f.close()
def packData(data, options):
""" packData stores all of the data in the appropriate pickle cache file.
"""
stats_file = getStatsFileName(options.jobTree)
cache_file = getStatsCacheFileName(options.jobTree)
try:
# try to write to the jobTree directory
payload = (data, stats_file)
f = open(cache_file, "wb")
cPickle.dump(payload, f, 2) # 2 is binary format
f.close()
except IOError:
if not options.cache:
return
# try to write to the current working directory only if --cache
cache_file = os.path.abspath(os.path.join(os.getcwd(),
".stats_cache.pickle"))
payload = (data, stats_file)
f = open(cache_file, "wb")
cPickle.dump(payload, f, 2) # 2 is binary format
f.close()