def get_eos_tot_size(top_dir): top_size = 0 subdir_sizes = {} iteration = 0 dirs, files, sizes = eosutil.parse_eos_dir(top_dir) # only iterate once here to # get the directories if dirs: for sdir in dirs: subdir_sizes[sdir] = 0 for stop, sdirs, sfiles, ssizes in eosutil.walk_eos(top_dir + '/' + sdir): for size in ssizes: subdir_sizes[sdir] = subdir_sizes[sdir] + size top_size = reduce(lambda x, y: x + y, subdir_sizes.values()) for size in sizes: top_size += size return top_size, subdir_sizes
def get_eos_tot_size( top_dir ) : top_size = 0 subdir_sizes = {} iteration = 0 dirs, files, sizes = eosutil.parse_eos_dir( top_dir ) # only iterate once here to # get the directories if dirs : for sdir in dirs : subdir_sizes[sdir] = 0 for stop, sdirs, sfiles, ssizes in eosutil.walk_eos( top_dir+'/'+sdir) : for size in ssizes : subdir_sizes[sdir] = subdir_sizes[sdir] + size top_size = reduce(lambda x, y : x+y , subdir_sizes.values() ) for size in sizes : top_size += size return top_size, subdir_sizes
def main() : if options.path is None : print 'Must provide a path' sys.exit(-1) top_dir = options.path.split('/')[-1] output_file = 'anaysis_config_all.txt' output_path='/tmp/jkunkle/%s/%s' %(top_dir, output_file) # gahter a list of files ana_files = [] if options.path.count( '/eos' ) : dirs, files, sizes = eosutil.parse_eos_dir( options.path ) for file in files : if file.count('.txt') : ana_files.append( options.path + '/' + file ) # if the output file is in the # list of input files, remove it # so it is not removed from # disk if len(ana_files) < 2 : print 'It looks like files have already been combined. Will abort.' return # sort the files so they have some reasonable ordering ana_files.sort() if output_file in ana_files : print 'Output file already in list of input files. Has the script already been run on this directory?' ana_files.remove( output_file ) print 'Will combine %d files' %len(ana_files) # copy files locally for filepath in ana_files : filename = filepath.split('/')[-1] eosutil.copy_eos_to_local( filepath, '/tmp/jkunkle/%s/%s' %(top_dir, filename) ) print 'files copied locally' ## cat the files for filepath in ana_files : filename = filepath.split('/')[-1] os.system( 'echo "%s" >> %s' %(filename, output_path) ) os.system( 'cat /tmp/jkunkle/%s/%s >> %s' %(top_dir, filename, output_path) ) os.system( 'echo "" >> %s' %(output_path) ) os.system( 'echo "" >> %s' %(output_path) ) print 'catted files' eosutil.copy_eos_to_local( output_path, options.path +'/' + output_file ) print 'copied output file to eos' # delete original file for filepath in ana_files : eosutil.rm_eos( filepath ) print 'deleted %d files from eos' %len(ana_files) print '\033[1;32m^.^ FINISHED ^.^\033[0m' else : wrapper_files = [] for top, dirs, files in os.walk( options.path ) : for file in files : if file.count('.txt') : ana_files.append( top + '/' + file ) elif file.count('wrapper') : wrapper_files.append( top + '/' + file ) # if the output file is in the # list of input files, remove it # so it is not removed from # disk if len(ana_files) < 2 : print 'It looks like files have already been combined. Will abort.' return # sort the files so they have some reasonable ordering ana_files.sort() print 'Will combine %d files' %len(ana_files) if output_file in ana_files : print 'Output file already in list of input files. Has the script already been run on this directory?' ana_files.remove( output_file ) # cat the files for filepath in ana_files : filename = filepath.split('/')[-1] os.system( 'echo "%s" >> %s/%s ' %( filename, options.path, output_file ) ) os.system( 'cat %s >> %s/%s' %(filepath, options.path, output_file) ) os.system( 'echo "" >> %s/%s' %(options.path, output_file) ) os.system( 'echo "" >> %s/%s' %(options.path, output_file) ) # delete original file for filepath in ana_files+wrapper_files : os.system( 'rm %s ' %filepath ) print 'deleted %d files ' %(len(ana_files)+len(wrapper_files)) print '\033[1;32m^.^ FINISHED ^.^\033[0m'