def main():
usage = "usage: %prog [options] dcache-dir-relative-to-user-dir"
optParser = optparse.OptionParser(usage)
optParser.add_option("-c", "--config", dest="cfgfile",
help="global configuration file",
default=ara.defaultConfigFileName)
(options, args) = optParser.parse_args()
if len(args) != 1:
optParser.print_help()
return 1
ara.config.read(options.cfgfile)
# get file names from Grid storage element
srmfiles = ara.uberftpls(ara.getUserStoragePath(args[0]), True)
# sum up size
totalsize = 0
for (size, filename) in srmfiles:
totalsize = totalsize + size
TB = int(totalsize/1000000000000)
GB = int(totalsize/1000000000) % 1000
MB = int(totalsize/1000000) % 1000
KB = int(totalsize/1000) % 1000
B = int(totalsize % 1000)
print "Found", len(srmfiles), "files:", TB, "TB", GB, "GB", MB, "MB", KB, "KB", B, "B"
return 0
def submit(jobgroup, job, period):
global options
global inbox
# read files from directory
# call program and record both stderr and stdout
filelist = []
section = jobgroup+':'+period
try:
for thisdir in ara.config.get(section, job).split(','):
print 'Adding files from directory', thisdir
# run over files on dCache?
if thisdir.startswith("dcap:/"):
# list files on dcache
path=thisdir[5:]
files = ara.uberftpls(path)
for (size, filename) in files:
filelist.append(ara.config.get('Grid', 'dcap') + path + '/' + filename)
else:
filespec = os.path.abspath(os.path.expanduser(thisdir)) + '/*.root'
files = ara.getCommandOutput2('ls ' + filespec)
for filename in files.splitlines():
filelist.append(filename)
except ConfigParser.NoSectionError:
print "Could not find section %s in main configuration file %s" % ( section , options.cfgfile )
print "You must create this section and add entries to it"
return False
# create directory
basedir = ara.config.get('Analysis', 'basedir').rstrip('/')
myDir = basedir + '/' + options.selection + '/' + period
mkdir_p(myDir)
# need to go into this directory - submit job from there, output will get here
os.chdir(myDir)
# determine file splitting
print 'Found', len(filelist), 'files'
list_of_lists = ara.split_in_jobs(filelist, options.nsplit)
n = 0
for files in list_of_lists:
print "Job #", n, ": ", len(files), " files"
executable = './analyzer'
jobName = job + '_' + str(n)
outputFile = jobName + '.root'
outbox = [ outputFile ]
arguments = job + " " + outputFile + " analyzer.cfg " + " ".join(files)
submit_condor_with_wrapper(executable, arguments,
",".join(inbox), ",".join(outbox),
jobName)
time.sleep(1)
# Next file
n += 1
return True
def main():
usage = """usage: %prog [options] dcache-dir
This program transfers files from the given directory on dcache to
the current local directory. It checks the time stamp and sizes of
all files, and only transfers those files which are newer or not
completely transferred."""
optParser = optparse.OptionParser(usage)
optParser.add_option("-c", "--config", dest="cfgfile",
help="global configuration file",
default=ara.defaultConfigFileName)
optParser.add_option("-f", "--force-deletion", action="store_true",
dest="forcedeletion",
help="force deletion of local files",
default=False)
(options, args) = optParser.parse_args()
ara.config.read(options.cfgfile)
# user must give path as argument
if len(args) != 1:
optParser.print_help()
return 1
# get file names from Grid storage element
print "Reading files from grid storage, please be patient..."
srcpath = ara.getUserStoragePath(args[0])
srmfiles = ara.uberftpls(srcpath, True)
# get local file names
print "Reading local file names and stats..."
filelist = os.listdir(".")
localfiles = [ ]
for localfile in filelist:
file_stats = os.stat(localfile)
if stat.S_ISREG(file_stats.st_mode):
size = file_stats.st_size
localfiles.append([size, localfile])
# loop over all grid files and see if they are here
print "Reconciling changes..."
transferfiles = [ ]
for srmfile in srmfiles:
if not srmfile[1].endswith(".root"):
print "Warning: Found non-ROOT file on grid storage:", srmfile
continue
try:
index = localfiles.index(srmfile)
except ValueError:
transferfiles.append(srmfile[1])
print "Need to transfer file", srmfile[1]
# determine if local ROOT files need to be deleted
deletefiles = [ ]
for localfile in localfiles:
if not localfile[1].endswith(".root"):
continue
try:
index = srmfiles.index(localfile)
except ValueError:
deletefiles.append(localfile[1])
for localfile in deletefiles:
if options.forcedeletion:
print "Deleting local file", localfile
os.unlink(localfile)
else:
print "Need to delete", localfile
if len(deletefiles) > 0 and not options.forcedeletion:
print "Use -f switch to delete files or remove files manually"
return 1
srcpath=ara.config.get('Grid', 'se')+srcpath
destpath = 'file:///'+os.getcwd()+'/'
for filename in transferfiles:
print filename
p = subprocess.Popen(["srmcp", srcpath+filename, destpath+filename],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if len(stderr) > 0:
print "Errors found:", stderr
return 1
if len(stdout) > 0:
print "Stdout", stdout
print "Done."
return 0