def master_configure(self): '''This method creates the tar.bz2 archive of user sw directory. Such a method is called one time per master job''' logger.debug('SBApp master_configure called.') self.now = datetime.datetime.now().strftime("%Y%m%d") self.os_arch = os.environ['SBROOT'].split('/')[-1] self.user_id = utils.getOwner() j = self.getJobObject() # check the target SE status using gridmon DB (updated by nagios monitoring system) sql = 'SELECT se_host, nagios_test_service FROM se WHERE name_grid = %s' local_SE = db.gridmon(sql, (getConfig('SuperB')['submission_site'], )) if local_SE[0]['nagios_test_service'] == 'CRITICAL': raise GangaException('Local storage element %s is down.' % local_SE[0]['se_host']) # logger.error('Local storage element %s seems died for gridmon.' % local_SE[0]['se_host']) #else: # logger.error('Local storage element %s is back alive for gridmon. !! uncomment exception !!' % local_SE[0]['se_host']) # create the software directory if self.software_dir != '': if not os.path.isdir(self.software_dir): raise ApplicationConfigurationError(None, 'software_dir must be a directory.') # make the tar file and update sw_archive parameter self.software_dir = os.path.normpath(self.software_dir) (head, tail) = os.path.split(self.software_dir) self.filename = tail self.sw_archive = os.path.join(j.inputdir, tail + '.tar.bz2') logger.info('Creating archive: %s ...', self.sw_archive) logger.info('From: %s', head) logger.info('Of: %s', tail) #savedir = os.getcwd() #os.chdir(self.software_dir) #retcode = subprocess.call("tar -cjf %s * 2>/dev/null" % self.sw_archive, shell=True) retcode = subprocess.call("tar -cjf %s -C %s %s 2>/dev/null" % (self.sw_archive, head, tail), shell=True) if retcode < 0: raise ApplicationConfigurationError(None, 'Error %d while creating archive.' % retcode) #os.chdir(savedir) else: raise ApplicationConfigurationError(None, 'software_dir cannot be empty.') if self.executable == '': raise ApplicationConfigurationError(None, 'executable cannot be empty.') # checking that j.inputdata is a valid object if not isinstance(j.inputdata, (SBInputDataset.SBInputPersonalProduction, SBInputDataset.SBInputProductionAnalysis, SBInputDataset.SBInputPureAnalysis)): msg = 'j.inputdata %s is not allowed' % str(type(j.inputdata)) raise ApplicationConfigurationError(None, msg) # checking that j.inputdata (the input dataset) is a valid dataset j.inputdata.check() # checking that j.outputdata (the output dataset) is valid if isinstance(j.outputdata, SBOutputDataset.SBOutputDataset): j.outputdata.check() # creating temp dataset self.temp_dataset = str(objectid.ObjectId()) free_string = '%s_%s_%s' % (j.id, j.name, self.filename) sql = '''INSERT INTO analysis_dataset (owner, dataset_id, session, parameters, status) VALUES (%s, decode(%s, 'hex'), %s, %s, 'temp'); INSERT INTO analysis_dataset_site (dataset_id, site) VALUES (decode(%s, 'hex'), %s);''' params = (utils.getOwner(), self.temp_dataset, 'analysis', {'free_string': free_string}, self.temp_dataset, getConfig('SuperB')['submission_site']) db.write(sql, params) # merger j.merger = TextMerger() j.merger.files.extend(['severus.log', 'output_files.txt']) j.merger.ignorefailed = True j.merger.compress = True j.splitter = SBSubmission.SBSubmission() return (0, None)
def master_configure(self): '''This method creates the tar.bz2 archive of user sw directory. Such a method is called one time per master job''' logger.debug('SBApp master_configure called.') self.now = datetime.datetime.now().strftime("%Y%m%d") self.os_arch = os.environ['SBROOT'].split('/')[-1] self.user_id = utils.getOwner() j = self.getJobObject() # check the target SE status using gridmon DB (updated by nagios monitoring system) sql = 'SELECT se_host, nagios_test_service FROM se WHERE name_grid = %s' local_SE = db.gridmon(sql, (getConfig('SuperB')['submission_site'], )) if local_SE[0]['nagios_test_service'] == 'CRITICAL': raise GangaException('Local storage element %s is down.' % local_SE[0]['se_host']) # logger.error('Local storage element %s seems died for gridmon.' % local_SE[0]['se_host']) #else: # logger.error('Local storage element %s is back alive for gridmon. !! uncomment exception !!' % local_SE[0]['se_host']) # create the software directory if self.software_dir != '': if not os.path.isdir(self.software_dir): raise ApplicationConfigurationError( 'software_dir must be a directory.') # make the tar file and update sw_archive parameter self.software_dir = os.path.normpath(self.software_dir) (head, tail) = os.path.split(self.software_dir) self.filename = tail self.sw_archive = os.path.join(j.inputdir, tail + '.tar.bz2') logger.info('Creating archive: %s ...', self.sw_archive) logger.info('From: %s', head) logger.info('Of: %s', tail) #savedir = os.getcwd() #os.chdir(self.software_dir) #retcode = subprocess.call("tar -cjf %s * 2>/dev/null" % self.sw_archive, shell=True) retcode = subprocess.call("tar -cjf %s -C %s %s 2>/dev/null" % (self.sw_archive, head, tail), shell=True) if retcode < 0: raise ApplicationConfigurationError( 'Error %d while creating archive.' % retcode) #os.chdir(savedir) else: raise ApplicationConfigurationError( 'software_dir cannot be empty.') if self.executable == '': raise ApplicationConfigurationError('executable cannot be empty.') # checking that j.inputdata is a valid object if not isinstance(j.inputdata, (SBInputDataset.SBInputPersonalProduction, SBInputDataset.SBInputProductionAnalysis, SBInputDataset.SBInputPureAnalysis)): msg = 'j.inputdata %s is not allowed' % str(type(j.inputdata)) raise ApplicationConfigurationError(msg) # checking that j.inputdata (the input dataset) is a valid dataset j.inputdata.check() # checking that j.outputdata (the output dataset) is valid if isinstance(j.outputdata, SBOutputDataset.SBOutputDataset): j.outputdata.check() # creating temp dataset self.temp_dataset = str(objectid.ObjectId()) free_string = '%s_%s_%s' % (j.id, j.name, self.filename) sql = '''INSERT INTO analysis_dataset (owner, dataset_id, session, parameters, status) VALUES (%s, decode(%s, 'hex'), %s, %s, 'temp'); INSERT INTO analysis_dataset_site (dataset_id, site) VALUES (decode(%s, 'hex'), %s);''' params = (utils.getOwner(), self.temp_dataset, 'analysis', { 'free_string': free_string }, self.temp_dataset, getConfig('SuperB')['submission_site']) db.write(sql, params) # merger j.merger = TextMerger() j.merger.files.extend(['severus.log', 'output_files.txt']) j.merger.ignorefailed = True j.merger.compress = True j.splitter = SBSubmission.SBSubmission() return (0, None)
def master_prepare(self, app, appmasterconfig): #Get the string jdl requirement per site and set it up for ganga job logger.debug('LCGRTHandler master_prepare called.') j = app._getParent() if len(j.inputdata.run_site) == 0: j.inputdata.getDefaultRunSite() # check CEs status of the chosen site using gridmon DB (updated by nagios monitoring system) sql = '''SELECT name_grid, status_other, banned FROM site WHERE false''' for site in j.inputdata.run_site: sql += ' OR name_grid = \'%s\' ' % site sql += 'ORDER BY name_grid' sites = db.gridmon(sql) for site in sites: if site['banned'] == True: logger.error('%s is not available due to technical problems.' % site['name_grid']) sites.remove(site) if site['status_other'] != 'OK': logger.error('%s is not available because of administrative policies.' % site['name_grid']) sites.remove(site) if len(sites) == 0: raise ApplicationConfigurationError(None, 'No sites available, try later again.') sql = '''SELECT ce_host FROM ce WHERE (nagios_test_service = 'OK' OR nagios_test_service = 'WARNING') AND (false''' for site in sites: sql += ' OR name_grid = \'%s\'' % site['name_grid'] sql += ' ) ORDER BY name_grid, ce_host' ces = db.gridmon(sql) requirements = 'False' for ce in ces: requirements += ' || ' requirements += '(other.GlueCEInfoHostName == "%s")' % ce['ce_host'] # converting from unicode string to normal python string (why is it necessary?) requirements = str(requirements) # cleaning useless flags requirements = requirements.replace('False || ', '') # debugging print logger.debug('Requirements: %s' % requirements) # if all sites are down and/or requirements are not defined, stop submission if requirements == 'False': raise ApplicationConfigurationError(None, 'Requirements are empty.') j.backend.requirements.other += [requirements] # setting WALLTIME (if the user hasn't setted it before) if j.backend.requirements.walltime == 0: j.backend.requirements.walltime = 24 * 60 # dobbiamo fornire un valore in minuti
def master_prepare(self, app, appmasterconfig): #Get the string jdl requirement per site and set it up for ganga job logger.debug('LCGRTHandler master_prepare called.') j = app._getParent() if len(j.inputdata.run_site) == 0: j.inputdata.getDefaultRunSite() # check CEs status of the chosen site using gridmon DB (updated by nagios monitoring system) sql = '''SELECT name_grid, status_other, banned FROM site WHERE false''' for site in j.inputdata.run_site: sql += ' OR name_grid = \'%s\' ' % site sql += 'ORDER BY name_grid' sites = db.gridmon(sql) for site in sites: if site['banned'] == True: logger.error('%s is not available due to technical problems.' % site['name_grid']) sites.remove(site) if site['status_other'] != 'OK': logger.error( '%s is not available because of administrative policies.' % site['name_grid']) sites.remove(site) if len(sites) == 0: raise ApplicationConfigurationError( 'No sites available, try later again.') sql = '''SELECT ce_host FROM ce WHERE (nagios_test_service = 'OK' OR nagios_test_service = 'WARNING') AND (false''' for site in sites: sql += ' OR name_grid = \'%s\'' % site['name_grid'] sql += ' ) ORDER BY name_grid, ce_host' ces = db.gridmon(sql) requirements = 'False' for ce in ces: requirements += ' || ' requirements += '(other.GlueCEInfoHostName == "%s")' % ce['ce_host'] # converting from unicode string to normal python string (why is it necessary?) requirements = str(requirements) # cleaning useless flags requirements = requirements.replace('False || ', '') # debugging print logger.debug('Requirements: %s' % requirements) # if all sites are down and/or requirements are not defined, stop submission if requirements == 'False': raise ApplicationConfigurationError('Requirements are empty.') j.backend.requirements.other += [requirements] # setting WALLTIME (if the user hasn't setted it before) if j.backend.requirements.walltime == 0: j.backend.requirements.walltime = 24 * 60 # dobbiamo fornire un valore in minuti