def runFiles(basedir):
ldir = [x[0] for x in os.walk(basedir)]
user = os.environ['USER']
exten = ''
for l in ldir:
All_files = glob.glob("%s/*events*" % (l))
if len(All_files) == 0: continue
process = l.split('/')[-1]
if process == '':
ltmp = l.split('/')
process = l.split('/')[len(ltmp) - 2]
print 'process ', process
for f in All_files:
if len(f.split('/')[-1]) > 19 or 'events_' in f: continue
if not os.path.isfile(f): continue
if '.root' in f:
exten = '.root'
elif '.lhe.gz' in f:
exten = '.lhe.gz'
ori_id = f.split('/')[-1].replace('events', '')
ori_id = ori_id.split('.')[0]
if user in ut.find_owner(f) or user == 'helsens':
baseid = ''
ori_id = int(ori_id)
ori_id += 1
for i in xrange(9 - len(str(ori_id))):
baseid += '0'
new_id = baseid + str(ori_id)
print ori_id, ' ', new_id
uniqueID = '%s' % (new_id)
outfile = 'events_%s%s' % (uniqueID, exten)
cmd = 'mv %s %s' % (f, l + '/' + outfile)
#cmdBAD = 'mv %s %s'%(f,basedir+'/'+outfile)
print cmd
os.system(cmd)
time.sleep(0.01)
def check(self, force, statfile):
#ldir=[x[0] for x in os.walk(self.indir)]
ldir = next(os.walk(self.indir))[1]
if not ut.testeos(self.para.eostest, self.para.eostest_size):
print 'eos seems to have problems, should check, will exit'
sys.exit(3)
for l in ldir:
if self.process != '' and self.process != l:
continue
#continue if process has been checked
if l == 'BADPYTHIA' or l == 'lhe' or l == "__restored_files__" or l == "backup":
continue
print '%s/%s/check' % (self.yamldir, l)
if not ut.file_exist('%s/%s/check' %
(self.yamldir, l)) and not force:
continue
print '--------------------- ', l
process = l
All_files = glob.glob("%s/%s/events_*%s" %
(self.indir, l, self.fext))
print 'number of files ', len(All_files)
if len(All_files) == 0: continue
print 'process from the input directory ', process
outdir = self.makeyamldir(self.yamldir + process)
hasbeenchecked = False
nevents_tot = 0
njobsdone_tot = 0
njobsbad_tot = 0
for f in All_files:
self.count = 0
if not os.path.isfile(f):
print 'file does not exists... %s' % f
continue
jobid = f.split('_')[-1]
jobid = jobid.replace(self.fext, '')
userid = ut.find_owner(f)
outfile = '%sevents_%s.yaml' % (outdir, jobid)
if ut.getsize(outfile) == 0:
cmd = "rm %s" % (outfile)
print 'file size 0, remove and continue ', cmd
os.system(cmd)
continue
if ut.file_exist(
outfile) and ut.getsize(outfile) > 100 and not force:
doc = None
with open(outfile) as ftmp:
try:
doc = yaml.load(ftmp)
except yaml.YAMLError as exc:
print(exc)
except IOError as exc:
print "I/O error({0}): {1}".format(
exc.errno, exc.strerror)
print "outfile ", outfile
try:
if doc != None: value = doc['processing']['status']
if value == 'DONE': continue
except KeyError, e:
print 'status %s does not exist' % str(e)
hasbeenchecked = True
print '-----------', f
if '.root' in self.fext:
nevts, check = self.checkFile_root(f, self.para.treename)
status = 'DONE'
if not check: status = 'BAD'
if status == 'DONE':
nevents_tot += nevts
njobsdone_tot += 1
else:
njobsbad_tot += 1
dic = {
'processing': {
'process': process,
'jobid': jobid,
'nevents': nevts,
'status': status,
'out': f,
'size': os.path.getsize(f),
'user': userid
}
}
try:
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
continue
except IOError as exc:
print "I/O error({0}): {1}".format(
exc.errno, exc.strerror)
print "outfile ", outfile
time.sleep(10)
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
continue
elif '.lhe.gz' in self.fext:
nevts, check = self.checkFile_lhe(f)
while nevts == -1 and not check:
nevts, check = self.checkFile_lhe(f)
if self.count == 10:
print 'can not copy or unzip the file, declare it wrong'
break
status = 'DONE'
if not check: status = 'BAD'
if status == 'DONE':
nevents_tot += nevts
njobsdone_tot += 1
else:
njobsbad_tot += 1
dic = {
'processing': {
'process': process,
'jobid': jobid,
'nevents': nevts,
'status': status,
'out': f,
'size': os.path.getsize(f),
'user': userid
}
}
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
continue
else:
print 'not correct file extension %s' % self.fext
if hasbeenchecked:
cmdp = '<pre>date=%s \t time=%s njobs=%i \t nevents=%i \t njobbad=%i \t process=%s </pre>\n' % (
ut.getdate_str(), ut.gettime_str(), njobsdone_tot,
nevents_tot, njobsbad_tot, process)
stat_exist = ut.file_exist(statfile)
with open(statfile, "a") as myfile:
if not stat_exist:
myfile.write(
'<link href="/afs/cern.ch/user/h/helsens/www/style/txtstyle.css" rel="stylesheet" type="text/css" />\n'
)
myfile.write(
'<style type="text/css"> /*<![CDATA[*/ .espace{ margin-left:3em } .espace2{ margin-top:9em } /*]]>*/ </style>\n'
)
myfile.write(cmdp)
print 'date=%s time=%s njobs=%i nevents=%i njobbad=%i process=%s' % (
ut.getdate_str(), ut.gettime_str(), njobsdone_tot,
nevents_tot, njobsbad_tot, process)
def check(self, force, statfile):
#ldir=[x[0] for x in os.walk(self.indir)]
ldir = next(os.walk(self.indir))[1]
if not ut.testeos(self.para.eostest, self.para.eostest_size):
print 'eos seems to have problems, should check, will exit'
sys.exit(3)
for l in ldir:
if self.process != '' and self.process != l:
continue
#continue if process has been checked
if ut.yamlcheck(self.yamlcheck, l) and not force: continue
print '--------------------- ', l
process = l
All_files = glob.glob("%s/%s/events_*%s" %
(self.indir, l, self.fext))
print 'number of files ', len(All_files)
if len(All_files) == 0: continue
if l == 'lhe' or l == "__restored_files__": continue
print 'process from the input directory ', process
outdir = self.makeyamldir(self.yamldir + process)
hasbeenchecked = False
nevents_tot = 0
njobsdone_tot = 0
njobsbad_tot = 0
for f in All_files:
self.count = 0
if not os.path.isfile(f):
print 'file does not exists... %s' % f
continue
jobid = f.split('_')[-1]
jobid = jobid.replace(self.fext, '')
userid = ut.find_owner(f)
outfile = '%sevents_%s.yaml' % (outdir, jobid)
if ut.file_exist(
outfile) and ut.getsize(outfile) > 100 and not force:
continue
hasbeenchecked = True
print '-----------', f
if '.root' in self.fext:
nevts, check = self.checkFile_root(f, self.para.treename)
status = 'DONE'
if not check: status = 'BAD'
if status == 'DONE':
nevents_tot += nevts
njobsdone_tot += 1
else:
njobsbad_tot += 1
dic = {
'processing': {
'process': process,
'jobid': jobid,
'nevents': nevts,
'status': status,
'out': f,
'size': os.path.getsize(f),
'user': userid
}
}
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
continue
elif '.lhe.gz' in self.fext:
nevts, check = self.checkFile_lhe(f)
while nevts == -1 and not check:
nevts, check = self.checkFile_lhe(f)
if self.count == 10:
print 'can not copy or unzip the file, declare it wrong'
break
status = 'DONE'
if not check: status = 'BAD'
if status == 'DONE':
nevents_tot += nevts
njobsdone_tot += 1
else:
njobsbad_tot += 1
dic = {
'processing': {
'process': process,
'jobid': jobid,
'nevents': nevts,
'status': status,
'out': f,
'size': os.path.getsize(f),
'user': userid
}
}
with open(outfile, 'w') as outyaml:
yaml.dump(dic, outyaml, default_flow_style=False)
continue
else:
print 'not correct file extension %s' % self.fext
if hasbeenchecked:
ut.yamlstatus(self.yamlcheck, process, False)
cmdp = 'date=%s <span class="espace"/> time=%s <span class="espace"/> njobs=%i <span class="espace"/> nevents=%i <span class="espace"/> njobbad=%i <span class="espace"/> process=%s <br>\n' % (
ut.getdate_str(), ut.gettime_str(), njobsdone_tot,
nevents_tot, njobsbad_tot, process)
stat_exist = ut.file_exist(statfile)
with open(statfile, "a") as myfile:
if not stat_exist:
myfile.write(
'<link href="/afs/cern.ch/user/h/helsens/www/style/txtstyle.css" rel="stylesheet" type="text/css" />\n'
)
myfile.write(
'<style type="text/css"> /*<![CDATA[*/ .espace{ margin-left:3em } .espace2{ margin-top:9em } /*]]>*/ </style>\n'
)
myfile.write(cmdp)
print 'date=%s time=%s njobs=%i nevents=%i njobbad=%i process=%s' % (
ut.getdate_str(), ut.gettime_str(), njobsdone_tot,
nevents_tot, njobsbad_tot, process)