def cleanoldjobs(self):
ldir = []
if self.process == '':
ldir = next(os.walk(self.yamldir))[1]
else:
ldir = [self.process]
#ldir=[x[0] for x in os.walk(self.yamldir)]
print ldir
for l in ldir:
All_files = []
if self.process == '':
All_files = glob.glob("%s/%s/events_*.yaml" %
(self.yamldir, l))
else:
All_files = glob.glob("%s/events_*.yaml" % (self.yamldir))
if len(All_files) == 0: continue
process = l
if self.process != '' and self.process != process:
continue
print 'process from the input directory ', process
for f in All_files:
if not os.path.isfile(f):
print 'file does not exists... %s' % f
continue
with open(f, 'r') as stream:
try:
tmpf = yaml.load(stream)
if tmpf['processing']['status'] == 'sending':
if ut.gettimestamp(
) - tmpf['processing']['timestamp'] > 6000:
print 'job %s is running since too long, will delete the yaml' % (
f)
#cmd="rm %s"%(tmpf['processing']['out'])
#print cmd
#os.system(cmd)
#cmd="rm %s"%(f)
#print cmd
#os.system(cmd)
except yaml.YAMLError as exc:
print(exc)
def makeyaml(outdir, uid):
if not ut.dir_exist(outdir):
os.system("mkdir %s" % outdir)
if outdir[-1] != '/':
outdir += '/'
outfile = '%sevents_%s.yaml' % (outdir, uid)
if ut.file_exist(outfile): return False
data = {
'processing': {
'status': 'sending',
'timestamp': ut.gettimestamp(),
}
}
with open(outfile, 'w') as outyaml:
yaml.dump(data, outyaml, default_flow_style=False)
return True
def cleanoldjobs(self):
ldir = []
if self.process == '':
ldir = next(os.walk(self.yamldir))[1]
else:
ldir = [self.process]
#ldir=[x[0] for x in os.walk(self.yamldir)]
print ldir
for l in ldir:
All_files = []
if self.process == '':
All_files = glob.glob("%s/%s/events_*.yaml" %
(self.yamldir, l))
else:
All_files = glob.glob("%s/events_*.yaml" % (self.yamldir))
print "%s/events_*.yaml" % (self.yamldir)
if len(All_files) == 0: continue
process = l
if self.process != '' and self.process != process:
print process
continue
print 'process from the input directory ', process
for f in All_files:
if not os.path.isfile(f):
print 'file does not exists... %s' % f
continue
if ut.getsize(f) == 0:
print 'file size 0 %s will delete the yaml' % (f)
cmd = "rm %s" % (f)
print cmd
os.system(cmd)
continue
with open(f, 'r') as stream:
try:
tmpf = yaml.load(stream)
if tmpf['processing']['status'] == 'sending':
#from datetime import datetime
#import time
#ts = time.time()
#print(ts)
#ds=str(tmpf['processing']['timestamp'])
#d = datetime(int(ds[0:4]), int(ds[5:6]), int(ds[7:8]), int(ds[9:10]), int(ds[11:12]))
#print int(ds[0:4]), int(ds[4:6]), int(ds[6:8]), int(ds[8:10]), int(ds[10:12])
#dt=datetime.timestamp()
#print dt
if ut.gettimestamp(
) - tmpf['processing']['timestamp'] > 18000:
print 'job %s is running since too long %i , will delete the yaml' % (
f, ut.gettimestamp() -
tmpf['processing']['timestamp'])
cmd = "rm %s" % (f)
print cmd
os.system(cmd)
except yaml.YAMLError as exc:
print(exc)
except IOError as e:
print(e)