def get_md5sum(fname, ignoreGzipTimestamp=False):
''' Calculates the MD5 checksum of a file '''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
## if the file is a zipped format (determined by extension),
## try to get checksum from it's content. The reason is that
## gzip file contains a timestamp in the header, which causes
## different md5sum value even the contents are the same.
#re_gzipfile = re.compile('.*[\.tgz|\.gz].*$')
f = None
if ignoreGzipTimestamp and (fname.find('.tgz') > 0 or fname.find('.gz') > 0):
f = gzip.open(fname,'rb')
else:
f = open(fname, 'rb')
m = get_md5_obj()
while True:
d = f.read(8096)
if not d:
break
m.update(d)
f.close()
md5sum = m.hexdigest()
profiler.check('md5sum calculation time')
return md5sum
def master_resubmit(self, rjobs):
'''Resubmit the master job to the grid'''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
job = self.getJobObject()
ick = False
if not job.master and len(job.subjobs) == 0:
# case 1: master job normal resubmission
logger.debug('rjobs: %s' % str(rjobs))
logger.debug('mode: master job normal resubmission')
ick = IBackend.master_resubmit(self, rjobs)
elif job.master:
# case 2: individual subjob resubmission
logger.debug('mode: individual subjob resubmission')
ick = IBackend.master_resubmit(self, rjobs)
else:
# case 3: master job bulk resubmission
logger.debug('mode: master job resubmission')
ick = self.master_bulk_resubmit(rjobs)
if not ick:
raise GangaException('ARC bulk submission failure')
profiler.check('job re-submission elapsed time')
return ick
def master_submit(self, rjobs, subjobconfigs, masterjobconfig):
'''Submit the master job to the grid'''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
job = self.getJobObject()
# finding CREAM CE endpoint for job submission
allowed_celist = []
try:
allowed_celist = self.requirements.getce()
if not self.CE and allowed_celist:
self.CE = allowed_celist[0]
except:
logger.warning(
'CREAM CE assigment from AtlasCREAMRequirements failed.')
if self.CE and allowed_celist:
if self.CE not in allowed_celist:
logger.warning('submission to CE not allowed: %s, use %s instead' % (
self.CE, allowed_celist[0]))
self.CE = allowed_celist[0]
if not self.CE:
raise GangaException('CREAM CE endpoint not set')
# delegate proxy to CREAM CE
self.delegation_id = Grid.cream_proxy_delegation(self.CE, self.delegation_id)
if not self.delegation_id:
logger.warning('proxy delegation to %s failed' % self.CE)
# doing massive job preparation
if len(job.subjobs) == 0:
ick = IBackend.master_submit(
self, rjobs, subjobconfigs, masterjobconfig)
else:
ick = self.master_bulk_submit(
rjobs, subjobconfigs, masterjobconfig)
profiler.check('==> master_submit() elapsed time')
return ick
def get_md5sum(fname, ignoreGzipTimestamp=False):
''' Calculates the MD5 checksum of a file '''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
# if the file is a zipped format (determined by extension),
# try to get checksum from it's content. The reason is that
# gzip file contains a timestamp in the header, which causes
# different md5sum value even the contents are the same.
#re_gzipfile = re.compile('.*[\.tgz|\.gz].*$')
f = None
if ignoreGzipTimestamp and (fname.find('.tgz') > 0
or fname.find('.gz') > 0):
f = gzip.open(fname, 'rb')
else:
f = open(fname, 'rb')
m = hashlib.md5()
while True:
d = f.read(8096)
if not d:
break
m.update(d)
f.close()
md5sum = m.hexdigest()
profiler.check('md5sum calculation time')
return md5sum
def master_resubmit(self, rjobs):
'''Resubmit the master job to the grid'''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
job = self.getJobObject()
ick = False
if not job.master and len(job.subjobs) == 0:
# case 1: master job normal resubmission
logger.debug('rjobs: %s' % str(rjobs))
logger.debug('mode: master job normal resubmission')
ick = IBackend.master_resubmit(self, rjobs)
elif job.master:
# case 2: individual subjob resubmission
logger.debug('mode: individual subjob resubmission')
ick = IBackend.master_resubmit(self, rjobs)
else:
# case 3: master job bulk resubmission
logger.debug('mode: master job resubmission')
ick = self.master_bulk_resubmit(rjobs)
if not ick:
raise GangaException('ARC bulk submission failure')
profiler.check('job re-submission elapsed time')
return ick
def master_submit(self, rjobs, subjobconfigs, masterjobconfig):
"""Submit the master job to the grid"""
profiler = ElapsedTimeProfiler(getLogger(name="Profile.LCG"))
profiler.start()
job = self.getJobObject()
# finding CREAM CE endpoint for job submission
allowed_celist = []
try:
allowed_celist = self.requirements.getce()
if not self.CE and allowed_celist:
self.CE = allowed_celist[0]
except:
logger.warning("CREAM CE assigment from AtlasCREAMRequirements failed.")
if self.CE and allowed_celist:
if self.CE not in allowed_celist:
logger.warning("submission to CE not allowed: %s, use %s instead" % (self.CE, allowed_celist[0]))
self.CE = allowed_celist[0]
if not self.CE:
raise GangaException("CREAM CE endpoint not set")
# delegate proxy to CREAM CE
if not grids["GLITE"].cream_proxy_delegation(self.CE):
logger.warning("proxy delegation to %s failed" % self.CE)
# doing massive job preparation
if len(job.subjobs) == 0:
ick = IBackend.master_submit(self, rjobs, subjobconfigs, masterjobconfig)
else:
ick = self.master_bulk_submit(rjobs, subjobconfigs, masterjobconfig)
profiler.check("==> master_submit() elapsed time")
return ick
def master_resubmit(self, rjobs):
'''Resubmit the master job to the grid'''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
job = self.getJobObject()
ick = False
# delegate proxy to CREAM CE
self.delegation_id = Grid.cream_proxy_delegation(self.CE, self.delegation_id)
if not self.delegation_id:
logger.warning('proxy delegation to %s failed' % self.CE)
if not job.master and len(job.subjobs) == 0:
# case 1: master job normal resubmission
logger.debug('rjobs: %s' % str(rjobs))
logger.debug('mode: master job normal resubmission')
ick = IBackend.master_resubmit(self, rjobs)
elif job.master:
# case 2: individual subjob resubmission
logger.debug('mode: individual subjob resubmission')
ick = IBackend.master_resubmit(self, rjobs)
else:
# case 3: master job bulk resubmission
logger.debug('mode: master job resubmission')
ick = self.master_bulk_resubmit(rjobs)
if not ick:
raise GangaException('CREAM bulk submission failure')
profiler.check('job re-submission elapsed time')
return ick
def master_resubmit(self, rjobs):
"""Resubmit the master job to the grid"""
profiler = ElapsedTimeProfiler(getLogger(name="Profile.LCG"))
profiler.start()
job = self.getJobObject()
ick = False
# delegate proxy to CREAM CE
if not grids["GLITE"].cream_proxy_delegation(self.CE):
logger.warning("proxy delegation to %s failed" % self.CE)
if not job.master and len(job.subjobs) == 0:
# case 1: master job normal resubmission
logger.debug("rjobs: %s" % str(rjobs))
logger.debug("mode: master job normal resubmission")
ick = IBackend.master_resubmit(self, rjobs)
elif job.master:
# case 2: individual subjob resubmission
logger.debug("mode: individual subjob resubmission")
ick = IBackend.master_resubmit(self, rjobs)
else:
# case 3: master job bulk resubmission
logger.debug("mode: master job resubmission")
ick = self.master_bulk_resubmit(rjobs)
if not ick:
raise GangaException("CREAM bulk submission failure")
profiler.check("job re-submission elapsed time")
return ick
def master_submit(self, rjobs, subjobconfigs, masterjobconfig):
'''Submit the master job to the grid'''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
job = self.getJobObject()
# finding ARC CE endpoint for job submission
#allowed_celist = []
# try:
# allowed_celist = self.requirements.getce()
# if not self.CE and allowed_celist:
# self.CE = allowed_celist[0]
# except:
# logger.warning('ARC CE assigment from ARCRequirements failed.')
# if self.CE and allowed_celist:
# if self.CE not in allowed_celist:
# logger.warning('submission to CE not allowed: %s, use %s instead' % ( self.CE, allowed_celist[0] ) )
# self.CE = allowed_celist[0]
# use arc info to check for any endpoints recorded in the config file
rc, output = grids['GLITE'].arc_info()
if not self.CE and rc != 0:
raise GangaException(
"ARC CE endpoint not set and no default settings in '%s'. " % config['ArcConfigFile'])
elif self.CE:
logger.info('ARC CE endpoint set to: ' + str(self.CE))
else:
logger.info("Using ARC CE endpoints defined in '%s'" %
config['ArcConfigFile'])
# delegate proxy to ARC CE
# if not grids['GLITE'].arc_proxy_delegation(self.CE):
# logger.warning('proxy delegation to %s failed' % self.CE)
# doing massive job preparation
if len(job.subjobs) == 0:
ick = IBackend.master_submit(
self, rjobs, subjobconfigs, masterjobconfig)
else:
ick = self.master_bulk_submit(
rjobs, subjobconfigs, masterjobconfig)
profiler.check('==> master_submit() elapsed time')
return ick
def master_submit(self, rjobs, subjobconfigs, masterjobconfig):
'''Submit the master job to the grid'''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
job = self.getJobObject()
# finding CREAM CE endpoint for job submission
allowed_celist = []
try:
allowed_celist = self.requirements.getce()
if not self.CE and allowed_celist:
self.CE = allowed_celist[0]
except:
logger.warning(
'CREAM CE assigment from AtlasCREAMRequirements failed.')
if self.CE and allowed_celist:
if self.CE not in allowed_celist:
logger.warning(
'submission to CE not allowed: %s, use %s instead' %
(self.CE, allowed_celist[0]))
self.CE = allowed_celist[0]
if not self.CE:
raise GangaException('CREAM CE endpoint not set')
# delegate proxy to CREAM CE
self.delegation_id = Grid.cream_proxy_delegation(
self.CE, self.delegation_id, self.credential_requirements)
if not self.delegation_id:
logger.warning('proxy delegation to %s failed' % self.CE)
# doing massive job preparation
if len(job.subjobs) == 0:
ick = IBackend.master_submit(self, rjobs, subjobconfigs,
masterjobconfig)
else:
ick = self.master_bulk_submit(rjobs, subjobconfigs,
masterjobconfig)
profiler.check('==> master_submit() elapsed time')
return ick
def master_resubmit(self, rjobs):
'''Resubmit the master job to the grid'''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
job = self.getJobObject()
ick = False
# delegate proxy to CREAM CE
self.delegation_id = Grid.cream_proxy_delegation(
self.CE, self.delegation_id, self.credential_requirements)
if not self.delegation_id:
logger.warning('proxy delegation to %s failed' % self.CE)
if not job.master and len(job.subjobs) == 0:
# case 1: master job normal resubmission
logger.debug('rjobs: %s' % str(rjobs))
logger.debug('mode: master job normal resubmission')
ick = IBackend.master_resubmit(self, rjobs)
elif job.master:
# case 2: individual subjob resubmission
logger.debug('mode: individual subjob resubmission')
ick = IBackend.master_resubmit(self, rjobs)
else:
# case 3: master job bulk resubmission
logger.debug('mode: master job resubmission')
ick = self.master_bulk_resubmit(rjobs)
if not ick:
raise GangaException('CREAM bulk submission failure')
profiler.check('job re-submission elapsed time')
return ick
