def cacheCopy(src, dst, min_free, file_map, no_delete):
#Cache a copy of src if possible, removing old files from cache if necessary
src_size = os.stat(src).st_size * 2 #Safety factor of 2 to account for file growth if cached copy is modified
du = os.statvfs(utilities.fullPath("/scratch/babymaker"))
avail = du.f_bsize*du.f_bavail
while avail-src_size < min_free:
#Keep deleting until there's room
if no_delete: return
removed_file = removeOldCache(file_map)
if not removed_file: return
du = os.statvfs(utilities.fullPath("/scratch/babymaker"))
avail = du.f_bsize*du.f_bavail
print("Caching "+src+" to "+dst+"\n")
try:
shutil.copy(src, dst)
os.chmod(dst, 0775)
while not cacheUpToDate(dst, src):
now = time.time()
os.utime(dst, (now, now))
except:
os.remove(dst)
utilities.ePrint("Failed to cache "+src+" to "+dst+"\n")
raise
def cacheRecurse(caches, file_map, command, fragile, min_free, no_delete):
if len(caches)==0:
#Caching done, run exectuable
execute(command, file_map, fragile)
return
net_path = caches[0]
if not isNetFile(net_path):
utilities.ePrint("Cannot cache "+net_path+"\n")
cacheRecurse(caches[1:], file_map, command, fragile, min_free, no_delete)
return
mkdirPermissions(os.path.dirname(net_path), 0775)
if not os.path.exists(net_path):
#If /net file does not exist, create new empty file
with open(net_path, "a"):
pass
cache_path = cachePath(net_path)
mkdirPermissions(os.path.dirname(cache_path), 0775)
if not cacheUpToDate(cache_path, net_path):
#Cache doesn't exist or is outdated, so copy file from /net
cacheCopy(net_path, cache_path, min_free, file_map, no_delete)
if cacheUpToDate(cache_path, net_path):
#Only use cached file if it was created and up-to-date
file_map[net_path] = cache_path
cacheRecurse(caches[1:], file_map, command, fragile, min_free, no_delete)
def cacheRecurse(caches, file_map, command, fragile, min_free, no_delete):
if len(caches) == 0:
#Caching done, run exectuable
execute(command, file_map, fragile)
return
net_path = caches[0]
if not isNetFile(net_path):
utilities.ePrint("Cannot cache " + net_path + "\n")
cacheRecurse(caches[1:], file_map, command, fragile, min_free,
no_delete)
return
mkdirPermissions(os.path.dirname(net_path), 0775)
if not os.path.exists(net_path):
#If /net file does not exist, create new empty file
with open(net_path, "a"):
pass
cache_path = cachePath(net_path)
mkdirPermissions(os.path.dirname(cache_path), 0775)
if not cacheUpToDate(cache_path, net_path):
#Cache doesn't exist or is outdated, so copy file from /net
cacheCopy(net_path, cache_path, min_free, file_map, no_delete)
if cacheUpToDate(cache_path, net_path):
#Only use cached file if it was created and up-to-date
file_map[net_path] = cache_path
cacheRecurse(caches[1:], file_map, command, fragile, min_free, no_delete)
def cacheCopy(src, dst, min_free, file_map, no_delete):
#Cache a copy of src if possible, removing old files from cache if necessary
src_size = os.stat(
src
).st_size * 2 #Safety factor of 2 to account for file growth if cached copy is modified
du = os.statvfs(utilities.fullPath("/scratch/babymaker"))
avail = du.f_bsize * du.f_bavail
while avail - src_size < min_free:
#Keep deleting until there's room
if no_delete: return
removed_file = removeOldCache(file_map)
if not removed_file: return
du = os.statvfs(utilities.fullPath("/scratch/babymaker"))
avail = du.f_bsize * du.f_bavail
print("Caching " + src + " to " + dst + "\n")
try:
shutil.copy(src, dst)
os.chmod(dst, 0775)
while not cacheUpToDate(dst, src):
now = time.time()
os.utime(dst, (now, now))
except:
os.remove(dst)
utilities.ePrint("Failed to cache " + src + " to " + dst + "\n")
raise
def getRules(slim_file_name):
rules = [ line.strip().split() for line in open(slim_file_name) ]
good_rules = [ rule for rule in rules
if len(rule)==0
or (len(rule)>0 and rule[0].startswith("#"))
or (len(rule)>=2 and (rule[0]=="keep" or rule[0]=="drop")) ]
bad_rules = [ rule for rule in rules if rule not in good_rules ]
good_rules = [ rule for rule in good_rules if len(rule)>=2 ]
for rule in bad_rules:
utilities.ePrint("Invalid rule:",rule,"\n")
return good_rules
def getRules(slim_file_name):
rules = [line.strip().split() for line in open(slim_file_name)]
good_rules = [
rule for rule in rules
if len(rule) == 0 or (len(rule) > 0 and rule[0].startswith("#")) or (
len(rule) >= 2 and (rule[0] == "keep" or rule[0] == "drop"))
]
bad_rules = [rule for rule in rules if rule not in good_rules]
good_rules = [rule for rule in good_rules if len(rule) >= 2]
for rule in bad_rules:
utilities.ePrint("Invalid rule:", rule, "\n")
return good_rules
def syncCache(net_path, cache_path):
try:
now = time.time()
os.utime(cache_path, (now, now))
cache_m_time = os.path.getmtime(cache_path)
while not cacheUpToDate(cache_path, net_path):
#Make sure cache is newer
cache_m_time += 1.
now = max(cache_m_time, time.time())
os.utime(cache_path, (now, now))
except:
os.remove(cache_path)
utilities.ePrint("Failed to sync cache times")
raise
def syncCache(net_path, cache_path):
try:
now = time.time()
os.utime(cache_path, (now, now))
cache_m_time = os.path.getmtime(cache_path)
while not cacheUpToDate(cache_path, net_path):
#Make sure cache is newer
cache_m_time += 1.
now = max(cache_m_time, time.time())
os.utime(cache_path, (now, now))
except:
os.remove(cache_path)
utilities.ePrint("Failed to sync cache times")
raise
def netCopy(src, dst):
print("Copying "+src+" to "+dst+"\n")
try:
shutil.copy(src, dst)
while not cacheUpToDate(src, dst):
#Want cache to be newer so it's considered up to date
now = time.time()
os.utime(src, (now, now))
except:
try:
os.remove(dst)
finally:
os.remove(src)
utilities.ePrint("Failed to copy "+src+" to "+dst+"\n")
raise
def netCopy(src, dst):
print("Copying " + src + " to " + dst + "\n")
try:
shutil.copy(src, dst)
while not cacheUpToDate(src, dst):
#Want cache to be newer so it's considered up to date
now = time.time()
os.utime(src, (now, now))
except:
try:
os.remove(dst)
finally:
os.remove(src)
utilities.ePrint("Failed to copy " + src + " to " + dst + "\n")
raise
def execute(command, file_map, fragile):
inv_file_map = dict()
if not fragile:
command, inv_file_map = mapFiles(command, file_map)
else:
inv_file_map = dict(
(cached, net) for net, cached in file_map.iteritems())
if len(command) <= 0: return
args = ["run/wrapper.sh"]
for a in command:
args.append(a.lstrip())
command = args
try:
old_mod_times = dict()
before_time = round(time.time() - 2.)
# 2 second safety margin in case executable modifies within access
# time resolution (typically 1 second)
for f in inv_file_map.iterkeys():
os.utime(f, (before_time, before_time))
old_mod_times[f] = os.path.getmtime(f)
exit_code = 0
print("Executing", command, "\n")
utilities.flush()
try:
exit_code = subprocess.call(command)
except SignalError as e:
if e.signum != signal.SIGCLD and e.signum != signal.SIGCHLD:
raise e
utilities.flush()
if exit_code != 0:
raise Exception("Executable returned non-zero exit code.")
except:
for f in inv_file_map.iterkeys():
os.remove(f)
utilities.ePrint("Failed to execute", command, "\n")
raise
else:
for cache_path, net_path in inv_file_map.iteritems():
if os.path.getmtime(cache_path) > old_mod_times[cache_path]:
#Copy modified files back to /net
netCopy(cache_path, net_path)
else:
syncCache(net_path, cache_path)
def execute(command, file_map, fragile):
inv_file_map = dict()
if not fragile:
command, inv_file_map = mapFiles(command, file_map)
else:
inv_file_map = dict((cached,net) for net,cached in file_map.iteritems())
if len(command) <= 0: return
args = ["run/wrapper.sh"]
for a in command:
args.append(a.lstrip())
command = args
try:
old_mod_times = dict()
before_time = round(time.time()-2.)
# 2 second safety margin in case executable modifies within access
# time resolution (typically 1 second)
for f in inv_file_map.iterkeys():
os.utime(f, (before_time, before_time))
old_mod_times[f] = os.path.getmtime(f)
exit_code = 0
print("Executing",command,"\n")
utilities.flush()
try:
exit_code = subprocess.call(command)
except SignalError as e:
if e.signum != signal.SIGCLD and e.signum != signal.SIGCHLD:
raise e
utilities.flush()
if exit_code != 0:
raise Exception("Executable returned non-zero exit code.")
except:
for f in inv_file_map.iterkeys():
os.remove(f)
utilities.ePrint("Failed to execute",command,"\n")
raise
else:
for cache_path, net_path in inv_file_map.iteritems():
if os.path.getmtime(cache_path) > old_mod_times[cache_path]:
#Copy modified files back to /net
netCopy(cache_path, net_path)
else:
syncCache(net_path, cache_path)
def singleFileDelete(path):
dirname = os.path.dirname(path)
prefix = os.path.splitext(os.path.basename(path))[0]+"_TMP_"
with tempfile.NamedTemporaryFile(dir=dirname, prefix=prefix, suffix=".root") as tmp:
try:
with utilities.ROOTFile(path, "read") as orig:
tree = orig.Get("tree")
if not tree:
utilities.ePrint("Could not find tree in "+path+". Skipping.")
return
with utilities.ROOTFile(tmp.name, "recreate") as outfile:
clone = tree.CloneTree(-1, "fast")
clone.Write()
print "Deleting treeglobal from "+path
shutil.copy(tmp.name, path)
except (utilities.ROOTOpenError, utilities.NonROOTFileError) as r:
utilities.ePrint("Could not open "+path+". Skipping.")
return