def master_configure(self):
'''This method creates the tar.bz2 archive of user sw directory.
Such a method is called one time per master job'''
logger.debug('SBApp master_configure called.')
self.now = datetime.datetime.now().strftime("%Y%m%d")
self.os_arch = os.environ['SBROOT'].split('/')[-1]
self.user_id = utils.getOwner()
j = self.getJobObject()
# check the target SE status using gridmon DB (updated by nagios monitoring system)
sql = 'SELECT se_host, nagios_test_service FROM se WHERE name_grid = %s'
local_SE = db.gridmon(sql, (getConfig('SuperB')['submission_site'], ))
if local_SE[0]['nagios_test_service'] == 'CRITICAL':
raise GangaException('Local storage element %s is down.' % local_SE[0]['se_host'])
# logger.error('Local storage element %s seems died for gridmon.' % local_SE[0]['se_host'])
#else:
# logger.error('Local storage element %s is back alive for gridmon. !! uncomment exception !!' % local_SE[0]['se_host'])
# create the software directory
if self.software_dir != '':
if not os.path.isdir(self.software_dir):
raise ApplicationConfigurationError(None, 'software_dir must be a directory.')
# make the tar file and update sw_archive parameter
self.software_dir = os.path.normpath(self.software_dir)
(head, tail) = os.path.split(self.software_dir)
self.filename = tail
self.sw_archive = os.path.join(j.inputdir, tail + '.tar.bz2')
logger.info('Creating archive: %s ...', self.sw_archive)
logger.info('From: %s', head)
logger.info('Of: %s', tail)
#savedir = os.getcwd()
#os.chdir(self.software_dir)
#retcode = subprocess.call("tar -cjf %s * 2>/dev/null" % self.sw_archive, shell=True)
retcode = subprocess.call("tar -cjf %s -C %s %s 2>/dev/null" % (self.sw_archive, head, tail), shell=True)
if retcode < 0:
raise ApplicationConfigurationError(None, 'Error %d while creating archive.' % retcode)
#os.chdir(savedir)
else:
raise ApplicationConfigurationError(None, 'software_dir cannot be empty.')
if self.executable == '':
raise ApplicationConfigurationError(None, 'executable cannot be empty.')
# checking that j.inputdata is a valid object
if not isinstance(j.inputdata, (SBInputDataset.SBInputPersonalProduction,
SBInputDataset.SBInputProductionAnalysis,
SBInputDataset.SBInputPureAnalysis)):
msg = 'j.inputdata %s is not allowed' % str(type(j.inputdata))
raise ApplicationConfigurationError(None, msg)
# checking that j.inputdata (the input dataset) is a valid dataset
j.inputdata.check()
# checking that j.outputdata (the output dataset) is valid
if isinstance(j.outputdata, SBOutputDataset.SBOutputDataset):
j.outputdata.check()
# creating temp dataset
self.temp_dataset = str(objectid.ObjectId())
free_string = '%s_%s_%s' % (j.id, j.name, self.filename)
sql = '''INSERT INTO analysis_dataset
(owner, dataset_id, session, parameters, status)
VALUES (%s, decode(%s, 'hex'), %s, %s, 'temp');
INSERT INTO analysis_dataset_site
(dataset_id, site)
VALUES (decode(%s, 'hex'), %s);'''
params = (utils.getOwner(),
self.temp_dataset,
'analysis',
{'free_string': free_string},
self.temp_dataset,
getConfig('SuperB')['submission_site'])
db.write(sql, params)
# merger
j.merger = TextMerger()
j.merger.files.extend(['severus.log', 'output_files.txt'])
j.merger.ignorefailed = True
j.merger.compress = True
j.splitter = SBSubmission.SBSubmission()
return (0, None)
def master_configure(self):
'''This method creates the tar.bz2 archive of user sw directory.
Such a method is called one time per master job'''
logger.debug('SBApp master_configure called.')
self.now = datetime.datetime.now().strftime("%Y%m%d")
self.os_arch = os.environ['SBROOT'].split('/')[-1]
self.user_id = utils.getOwner()
j = self.getJobObject()
# check the target SE status using gridmon DB (updated by nagios monitoring system)
sql = 'SELECT se_host, nagios_test_service FROM se WHERE name_grid = %s'
local_SE = db.gridmon(sql, (getConfig('SuperB')['submission_site'], ))
if local_SE[0]['nagios_test_service'] == 'CRITICAL':
raise GangaException('Local storage element %s is down.' %
local_SE[0]['se_host'])
# logger.error('Local storage element %s seems died for gridmon.' % local_SE[0]['se_host'])
#else:
# logger.error('Local storage element %s is back alive for gridmon. !! uncomment exception !!' % local_SE[0]['se_host'])
# create the software directory
if self.software_dir != '':
if not os.path.isdir(self.software_dir):
raise ApplicationConfigurationError(
'software_dir must be a directory.')
# make the tar file and update sw_archive parameter
self.software_dir = os.path.normpath(self.software_dir)
(head, tail) = os.path.split(self.software_dir)
self.filename = tail
self.sw_archive = os.path.join(j.inputdir, tail + '.tar.bz2')
logger.info('Creating archive: %s ...', self.sw_archive)
logger.info('From: %s', head)
logger.info('Of: %s', tail)
#savedir = os.getcwd()
#os.chdir(self.software_dir)
#retcode = subprocess.call("tar -cjf %s * 2>/dev/null" % self.sw_archive, shell=True)
retcode = subprocess.call("tar -cjf %s -C %s %s 2>/dev/null" %
(self.sw_archive, head, tail),
shell=True)
if retcode < 0:
raise ApplicationConfigurationError(
'Error %d while creating archive.' % retcode)
#os.chdir(savedir)
else:
raise ApplicationConfigurationError(
'software_dir cannot be empty.')
if self.executable == '':
raise ApplicationConfigurationError('executable cannot be empty.')
# checking that j.inputdata is a valid object
if not isinstance(j.inputdata,
(SBInputDataset.SBInputPersonalProduction,
SBInputDataset.SBInputProductionAnalysis,
SBInputDataset.SBInputPureAnalysis)):
msg = 'j.inputdata %s is not allowed' % str(type(j.inputdata))
raise ApplicationConfigurationError(msg)
# checking that j.inputdata (the input dataset) is a valid dataset
j.inputdata.check()
# checking that j.outputdata (the output dataset) is valid
if isinstance(j.outputdata, SBOutputDataset.SBOutputDataset):
j.outputdata.check()
# creating temp dataset
self.temp_dataset = str(objectid.ObjectId())
free_string = '%s_%s_%s' % (j.id, j.name, self.filename)
sql = '''INSERT INTO analysis_dataset
(owner, dataset_id, session, parameters, status)
VALUES (%s, decode(%s, 'hex'), %s, %s, 'temp');
INSERT INTO analysis_dataset_site
(dataset_id, site)
VALUES (decode(%s, 'hex'), %s);'''
params = (utils.getOwner(), self.temp_dataset, 'analysis', {
'free_string': free_string
}, self.temp_dataset, getConfig('SuperB')['submission_site'])
db.write(sql, params)
# merger
j.merger = TextMerger()
j.merger.files.extend(['severus.log', 'output_files.txt'])
j.merger.ignorefailed = True
j.merger.compress = True
j.splitter = SBSubmission.SBSubmission()
return (0, None)
def master_prepare(self, app, appmasterconfig):
#Get the string jdl requirement per site and set it up for ganga job
logger.debug('LCGRTHandler master_prepare called.')
j = app._getParent()
if len(j.inputdata.run_site) == 0:
j.inputdata.getDefaultRunSite()
# check CEs status of the chosen site using gridmon DB (updated by nagios monitoring system)
sql = '''SELECT name_grid,
status_other,
banned
FROM site
WHERE false'''
for site in j.inputdata.run_site:
sql += ' OR name_grid = \'%s\' ' % site
sql += 'ORDER BY name_grid'
sites = db.gridmon(sql)
for site in sites:
if site['banned'] == True:
logger.error('%s is not available due to technical problems.' % site['name_grid'])
sites.remove(site)
if site['status_other'] != 'OK':
logger.error('%s is not available because of administrative policies.' % site['name_grid'])
sites.remove(site)
if len(sites) == 0:
raise ApplicationConfigurationError(None, 'No sites available, try later again.')
sql = '''SELECT ce_host
FROM ce
WHERE (nagios_test_service = 'OK' OR nagios_test_service = 'WARNING') AND (false'''
for site in sites:
sql += ' OR name_grid = \'%s\'' % site['name_grid']
sql += ' ) ORDER BY name_grid, ce_host'
ces = db.gridmon(sql)
requirements = 'False'
for ce in ces:
requirements += ' || '
requirements += '(other.GlueCEInfoHostName == "%s")' % ce['ce_host']
# converting from unicode string to normal python string (why is it necessary?)
requirements = str(requirements)
# cleaning useless flags
requirements = requirements.replace('False || ', '')
# debugging print
logger.debug('Requirements: %s' % requirements)
# if all sites are down and/or requirements are not defined, stop submission
if requirements == 'False':
raise ApplicationConfigurationError(None, 'Requirements are empty.')
j.backend.requirements.other += [requirements]
# setting WALLTIME (if the user hasn't setted it before)
if j.backend.requirements.walltime == 0:
j.backend.requirements.walltime = 24 * 60 # dobbiamo fornire un valore in minuti
def master_prepare(self, app, appmasterconfig):
#Get the string jdl requirement per site and set it up for ganga job
logger.debug('LCGRTHandler master_prepare called.')
j = app._getParent()
if len(j.inputdata.run_site) == 0:
j.inputdata.getDefaultRunSite()
# check CEs status of the chosen site using gridmon DB (updated by nagios monitoring system)
sql = '''SELECT name_grid,
status_other,
banned
FROM site
WHERE false'''
for site in j.inputdata.run_site:
sql += ' OR name_grid = \'%s\' ' % site
sql += 'ORDER BY name_grid'
sites = db.gridmon(sql)
for site in sites:
if site['banned'] == True:
logger.error('%s is not available due to technical problems.' %
site['name_grid'])
sites.remove(site)
if site['status_other'] != 'OK':
logger.error(
'%s is not available because of administrative policies.' %
site['name_grid'])
sites.remove(site)
if len(sites) == 0:
raise ApplicationConfigurationError(
'No sites available, try later again.')
sql = '''SELECT ce_host
FROM ce
WHERE (nagios_test_service = 'OK' OR nagios_test_service = 'WARNING') AND (false'''
for site in sites:
sql += ' OR name_grid = \'%s\'' % site['name_grid']
sql += ' ) ORDER BY name_grid, ce_host'
ces = db.gridmon(sql)
requirements = 'False'
for ce in ces:
requirements += ' || '
requirements += '(other.GlueCEInfoHostName == "%s")' % ce['ce_host']
# converting from unicode string to normal python string (why is it necessary?)
requirements = str(requirements)
# cleaning useless flags
requirements = requirements.replace('False || ', '')
# debugging print
logger.debug('Requirements: %s' % requirements)
# if all sites are down and/or requirements are not defined, stop submission
if requirements == 'False':
raise ApplicationConfigurationError('Requirements are empty.')
j.backend.requirements.other += [requirements]
# setting WALLTIME (if the user hasn't setted it before)
if j.backend.requirements.walltime == 0:
j.backend.requirements.walltime = 24 * 60 # dobbiamo fornire un valore in minuti
