def master_resubmit(self, rjobs): '''Resubmit the master job to the grid''' profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG')) profiler.start() job = self.getJobObject() ick = False if not job.master and len(job.subjobs) == 0: # case 1: master job normal resubmission logger.debug('rjobs: %s' % str(rjobs)) logger.debug('mode: master job normal resubmission') ick = IBackend.master_resubmit(self, rjobs) elif job.master: # case 2: individual subjob resubmission logger.debug('mode: individual subjob resubmission') ick = IBackend.master_resubmit(self, rjobs) else: # case 3: master job bulk resubmission logger.debug('mode: master job resubmission') ick = self.master_bulk_resubmit(rjobs) if not ick: raise GangaException('ARC bulk submission failure') profiler.check('job re-submission elapsed time') return ick
def get_md5sum(fname, ignoreGzipTimestamp=False): ''' Calculates the MD5 checksum of a file ''' profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG')) profiler.start() # if the file is a zipped format (determined by extension), # try to get checksum from it's content. The reason is that # gzip file contains a timestamp in the header, which causes # different md5sum value even the contents are the same. #re_gzipfile = re.compile('.*[\.tgz|\.gz].*$') f = None if ignoreGzipTimestamp and (fname.find('.tgz') > 0 or fname.find('.gz') > 0): f = gzip.open(fname, 'rb') else: f = open(fname, 'rb') m = hashlib.md5() while True: d = f.read(8096) if not d: break m.update(d) f.close() md5sum = m.hexdigest() profiler.check('md5sum calculation time') return md5sum
def get_md5sum(fname, ignoreGzipTimestamp=False): ''' Calculates the MD5 checksum of a file ''' profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG')) profiler.start() ## if the file is a zipped format (determined by extension), ## try to get checksum from it's content. The reason is that ## gzip file contains a timestamp in the header, which causes ## different md5sum value even the contents are the same. #re_gzipfile = re.compile('.*[\.tgz|\.gz].*$') f = None if ignoreGzipTimestamp and (fname.find('.tgz') > 0 or fname.find('.gz') > 0): f = gzip.open(fname,'rb') else: f = open(fname, 'rb') m = get_md5_obj() while True: d = f.read(8096) if not d: break m.update(d) f.close() md5sum = m.hexdigest() profiler.check('md5sum calculation time') return md5sum
def master_submit(self, rjobs, subjobconfigs, masterjobconfig): '''Submit the master job to the grid''' profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG')) profiler.start() job = self.getJobObject() # finding ARC CE endpoint for job submission #allowed_celist = [] # try: # allowed_celist = self.requirements.getce() # if not self.CE and allowed_celist: # self.CE = allowed_celist[0] # except: # logger.warning('ARC CE assigment from ARCRequirements failed.') # if self.CE and allowed_celist: # if self.CE not in allowed_celist: # logger.warning('submission to CE not allowed: %s, use %s instead' % ( self.CE, allowed_celist[0] ) ) # self.CE = allowed_celist[0] # use arc info to check for any endpoints recorded in the config file rc, output = grids['GLITE'].arc_info() if not self.CE and rc != 0: raise GangaException( "ARC CE endpoint not set and no default settings in '%s'. " % config['ArcConfigFile']) elif self.CE: logger.info('ARC CE endpoint set to: ' + str(self.CE)) else: logger.info("Using ARC CE endpoints defined in '%s'" % config['ArcConfigFile']) # delegate proxy to ARC CE # if not grids['GLITE'].arc_proxy_delegation(self.CE): # logger.warning('proxy delegation to %s failed' % self.CE) # doing massive job preparation if len(job.subjobs) == 0: ick = IBackend.master_submit( self, rjobs, subjobconfigs, masterjobconfig) else: ick = self.master_bulk_submit( rjobs, subjobconfigs, masterjobconfig) profiler.check('==> master_submit() elapsed time') return ick
def master_submit(self, rjobs, subjobconfigs, masterjobconfig): '''Submit the master job to the grid''' profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG')) profiler.start() job = self.getJobObject() # finding CREAM CE endpoint for job submission allowed_celist = [] try: allowed_celist = self.requirements.getce() if not self.CE and allowed_celist: self.CE = allowed_celist[0] except: logger.warning( 'CREAM CE assigment from AtlasCREAMRequirements failed.') if self.CE and allowed_celist: if self.CE not in allowed_celist: logger.warning( 'submission to CE not allowed: %s, use %s instead' % (self.CE, allowed_celist[0])) self.CE = allowed_celist[0] if not self.CE: raise GangaException('CREAM CE endpoint not set') # delegate proxy to CREAM CE self.delegation_id = Grid.cream_proxy_delegation( self.CE, self.delegation_id, self.credential_requirements) if not self.delegation_id: logger.warning('proxy delegation to %s failed' % self.CE) # doing massive job preparation if len(job.subjobs) == 0: ick = IBackend.master_submit(self, rjobs, subjobconfigs, masterjobconfig) else: ick = self.master_bulk_submit(rjobs, subjobconfigs, masterjobconfig) profiler.check('==> master_submit() elapsed time') return ick
def master_submit(self, rjobs, subjobconfigs, masterjobconfig): '''Submit the master job to the grid''' profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG')) profiler.start() job = self.getJobObject() # finding CREAM CE endpoint for job submission allowed_celist = [] try: allowed_celist = self.requirements.getce() if not self.CE and allowed_celist: self.CE = allowed_celist[0] except: logger.warning( 'CREAM CE assigment from AtlasCREAMRequirements failed.') if self.CE and allowed_celist: if self.CE not in allowed_celist: logger.warning('submission to CE not allowed: %s, use %s instead' % ( self.CE, allowed_celist[0])) self.CE = allowed_celist[0] if not self.CE: raise GangaException('CREAM CE endpoint not set') # delegate proxy to CREAM CE self.delegation_id = Grid.cream_proxy_delegation(self.CE, self.delegation_id) if not self.delegation_id: logger.warning('proxy delegation to %s failed' % self.CE) # doing massive job preparation if len(job.subjobs) == 0: ick = IBackend.master_submit( self, rjobs, subjobconfigs, masterjobconfig) else: ick = self.master_bulk_submit( rjobs, subjobconfigs, masterjobconfig) profiler.check('==> master_submit() elapsed time') return ick
def master_resubmit(self, rjobs): '''Resubmit the master job to the grid''' profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG')) profiler.start() job = self.getJobObject() ick = False # delegate proxy to CREAM CE self.delegation_id = Grid.cream_proxy_delegation( self.CE, self.delegation_id, self.credential_requirements) if not self.delegation_id: logger.warning('proxy delegation to %s failed' % self.CE) if not job.master and len(job.subjobs) == 0: # case 1: master job normal resubmission logger.debug('rjobs: %s' % str(rjobs)) logger.debug('mode: master job normal resubmission') ick = IBackend.master_resubmit(self, rjobs) elif job.master: # case 2: individual subjob resubmission logger.debug('mode: individual subjob resubmission') ick = IBackend.master_resubmit(self, rjobs) else: # case 3: master job bulk resubmission logger.debug('mode: master job resubmission') ick = self.master_bulk_resubmit(rjobs) if not ick: raise GangaException('CREAM bulk submission failure') profiler.check('job re-submission elapsed time') return ick
def master_submit(self, rjobs, subjobconfigs, masterjobconfig): """Submit the master job to the grid""" profiler = ElapsedTimeProfiler(getLogger(name="Profile.LCG")) profiler.start() job = self.getJobObject() # finding CREAM CE endpoint for job submission allowed_celist = [] try: allowed_celist = self.requirements.getce() if not self.CE and allowed_celist: self.CE = allowed_celist[0] except: logger.warning("CREAM CE assigment from AtlasCREAMRequirements failed.") if self.CE and allowed_celist: if self.CE not in allowed_celist: logger.warning("submission to CE not allowed: %s, use %s instead" % (self.CE, allowed_celist[0])) self.CE = allowed_celist[0] if not self.CE: raise GangaException("CREAM CE endpoint not set") # delegate proxy to CREAM CE if not grids["GLITE"].cream_proxy_delegation(self.CE): logger.warning("proxy delegation to %s failed" % self.CE) # doing massive job preparation if len(job.subjobs) == 0: ick = IBackend.master_submit(self, rjobs, subjobconfigs, masterjobconfig) else: ick = self.master_bulk_submit(rjobs, subjobconfigs, masterjobconfig) profiler.check("==> master_submit() elapsed time") return ick
def master_resubmit(self, rjobs): '''Resubmit the master job to the grid''' profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG')) profiler.start() job = self.getJobObject() ick = False # delegate proxy to CREAM CE self.delegation_id = Grid.cream_proxy_delegation(self.CE, self.delegation_id) if not self.delegation_id: logger.warning('proxy delegation to %s failed' % self.CE) if not job.master and len(job.subjobs) == 0: # case 1: master job normal resubmission logger.debug('rjobs: %s' % str(rjobs)) logger.debug('mode: master job normal resubmission') ick = IBackend.master_resubmit(self, rjobs) elif job.master: # case 2: individual subjob resubmission logger.debug('mode: individual subjob resubmission') ick = IBackend.master_resubmit(self, rjobs) else: # case 3: master job bulk resubmission logger.debug('mode: master job resubmission') ick = self.master_bulk_resubmit(rjobs) if not ick: raise GangaException('CREAM bulk submission failure') profiler.check('job re-submission elapsed time') return ick
def master_resubmit(self, rjobs): """Resubmit the master job to the grid""" profiler = ElapsedTimeProfiler(getLogger(name="Profile.LCG")) profiler.start() job = self.getJobObject() ick = False # delegate proxy to CREAM CE if not grids["GLITE"].cream_proxy_delegation(self.CE): logger.warning("proxy delegation to %s failed" % self.CE) if not job.master and len(job.subjobs) == 0: # case 1: master job normal resubmission logger.debug("rjobs: %s" % str(rjobs)) logger.debug("mode: master job normal resubmission") ick = IBackend.master_resubmit(self, rjobs) elif job.master: # case 2: individual subjob resubmission logger.debug("mode: individual subjob resubmission") ick = IBackend.master_resubmit(self, rjobs) else: # case 3: master job bulk resubmission logger.debug("mode: master job resubmission") ick = self.master_bulk_resubmit(rjobs) if not ick: raise GangaException("CREAM bulk submission failure") profiler.check("job re-submission elapsed time") return ick