def merge_root_files(srcdir, dstdir, size_max='2G', nfile_max=500):
file_list = []
size_list = []
for root, dirs, files in os.walk(srcdir):
for f in files:
s = os.path.getsize(os.path.join(root, f))
file_list.append(os.path.join(root, f))
size_list.append(s)
totalsize = sum(size_list)
sys.stdout.write('total of %s files, size %s.\n' % (len(file_list), size(totalsize)))
com_name = file_list[0].split('-')[0].split('/')[-1]
groups = group_files_by_size(file_list, size_max)
ngroups = len(groups)
for i in range(ngroups):
print i
sourcefiles = ' '.join(groups[i-1])
targetname = '%s_merged_%s.root' % (com_name, i+ngroups)
targetfile = check_and_join(dstdir, targetname)
cmd = 'hadd %s %s' %(targetfile, sourcefiles)
sys.stdout.write('merging %s of %s ...\n' %(i+1, ngroups))
sys.stdout.flush()
output = proc_cmd(cmd)
print output
def main():
args = sys.argv[1:]
if len(args) < 2:
return usage()
src = args[0]
dst = args[1]
sys.stdout.write('Scanning %s...\n' % src)
file_list = []
for root, dirs, files in os.walk(src):
for f in files:
if os.path.exists(os.path.join(root, f)):
if f.endswith('.slcio'):
file_list.append(os.path.join(root, f))
if len(args) == 2:
save_list_into_file(file_list, dst)
return
if len(args) == 3:
size = args[2]
sys.stdout.write('Size limit per list: %s\n' % size)
groups = group_files_by_size(file_list, size)
path, name = os.path.split(dst)
dst_com = name.split('.')[0]
dst_ext = name.split('.')[1]
n = 0
for file_list in groups:
n = n + 1
new_dst = os.path.join(path,
'%s_%s-%s.%s' % (dst_com, size, n, dst_ext))
save_list_into_file(file_list, new_dst)
def main():
args = sys.argv[1:]
if len(args) < 2:
return usage()
src = args[0]
dst = args[1]
sys.stdout.write('Scanning %s...\n' %src)
file_list = []
for root, dirs, files in os.walk(src):
for f in files:
file_list.append(os.path.join(root, f))
if len(args) == 2:
save_list_into_file(file_list, dst)
return
if len(args) == 3:
size = args[2]
sys.stdout.write('Size limit per list: %s\n' %size)
groups = group_files_by_size(file_list, size)
path, name = os.path.split(dst)
dst_com = name.split('.')[0]
dst_ext = name.split('.')[1]
n = 0
for file_list in groups:
n = n+1
new_dst = os.path.join(path, '%s_%s-%s.%s' %
(dst_com, size, n, dst_ext))
save_list_into_file(file_list, new_dst)
def merge_root_files(srcdir, dstdir, size_max='5G', nfile_max=500):
file_list = []
size_list = []
for root, dirs, files in os.walk(srcdir):
for f in files:
s = os.path.getsize(os.path.join(root, f))
file_list.append(os.path.join(root, f))
size_list.append(s)
totalsize = sum(size_list)
sys.stdout.write('total of %s files, size %s.\n' % (len(file_list), size(totalsize)))
com_name = file_list[0].split('-')[0].split('/')[-1]
groups = group_files_by_size(file_list, size_max)
ngroups = len(groups)
for i in range(ngroups):
print i
sourcefiles = ' '.join(groups[i-1])
targetname = '%s_merged_%s.root' % (com_name, i+ngroups)
targetfile = check_and_join(dstdir, targetname)
cmd = 'hadd %s %s' %(targetfile, sourcefiles)
sys.stdout.write('merging %s of %s ...\n' %(i+1, ngroups))
sys.stdout.flush()
output = proc_cmd(cmd)
print output
def get_4170MeV(input_dir,output_file,size):
src = input_dir
dst = output_file
sys.stdout.write('Scanning %s...\n' %src)
file_list = []
run_list = []
for run in range(34447,34478):
run_list.append(run)
for root, dirs, files in os.walk(src):
for f in files:
for c in run_list:
if str(c) in str(f):
file_list.append(os.path.join(root, f))
print len(file_list)
sys.stdout.write('Size limit per list: %s\n' %size)
groups = group_files_by_size(file_list, size)
path, name = os.path.split(dst)
dst_com = name.split('.')[0]
dst_ext = name.split('.')[1]
n = 0
for file_list in groups:
n = n+1
new_dst = os.path.join(path, '%s_%s-%s.%s' %
(dst_com, size, n, dst_ext))
save_list_into_file(file_list, new_dst)
def main():
args = sys.argv[1:]
if len(args) < 2:
return
tab = args[0]
dst = args[1]
#sys.stdout.write('Scanning %s...\n' %src)
table = open(tab, 'r')
ctime = os.path.getctime(args[0])
print(ctime)
for s_line in table:
if not s_line.startswith('#'):
l = [x.strip() for x in s_line.split(',')]
dname = l[0]
output_dir = dst + '/' + dname
os.makedirs(output_dir)
src = l[4]
name_tag = l[6]
file_list = []
for root, dirs, files in os.walk(src):
for f in files:
if os.path.exists(os.path.join(root, f)):
if f.startswith(name_tag):
if f.endswith('.slcio'):
file_time = os.path.getmtime(
os.path.join(root, f))
diff_day = (ctime - file_time) / 3600.0 / 24.0
#if diff_day > 1.0 : ## to avoid access to generating slcio files
if diff_day > 0.0:
file_list.append(os.path.join(root, f))
fname = l[0] ## filename = dirname for the moment !
if len(args) == 3:
size = args[2]
sys.stdout.write('Size limit per list: %s\n' % size)
groups = group_files_by_size(file_list, size)
dst_com = fname.split('.')[0]
dst_ext = 'txt'
n = 0
for file_list in groups:
n = n + 1
new_dst = os.path.join(
output_dir,
'%s_%s-%s.%s' % (dst_com, size, n, dst_ext))
save_list_into_file(file_list, new_dst)