def __init__(self, config, name):
cancelExecutor = CancelAndPurgeJobs(config, CREAM_CancelJobs(config),
CREAM_PurgeJobs(config))
GridWMS.__init__(self,
config,
name,
checkExecutor=CREAM_CheckJobs(config),
cancelExecutor=ChunkedExecutor(
config, 'cancel', cancelExecutor))
self._nJobsPerChunk = config.getInt('job chunk size',
10,
onChange=None)
self._submitExec = utils.resolveInstallPath('glite-ce-job-submit')
self._outputExec = utils.resolveInstallPath('glite-ce-job-output')
self._submitParams.update({
'-r': self._ce,
'--config-vo': self._configVO
})
self._outputRegex = r'.*For JobID \[(?P<rawId>\S+)\] output will be stored in the dir (?P<outputDir>.*)$'
self._useDelegate = False
if self._useDelegate is False:
self._submitParams.update({'-a': ' '})
def __init__(self, config, name):
cancel_executor = CancelAndPurgeJobs(config, CREAMCancelJobs(config),
CREAMPurgeJobs(config))
GridWMS.__init__(
self,
config,
name,
submit_exec=resolve_install_path('glite-ce-job-submit'),
output_exec=resolve_install_path('glite-ce-job-output'),
check_executor=CREAMCheckJobs(config),
cancel_executor=ChunkedExecutor(config, 'cancel', cancel_executor))
self._log.info("CreamWMS.__init__")
self._delegate_exec = resolve_install_path('glite-ce-delegate-proxy')
self._use_delegate = config.get_bool('try delegate',
True,
on_change=None)
self._chunk_size = config.get_int('job chunk size', 10, on_change=None)
self._submit_args_dict.update({
'-r': self._ce,
'--config-vo': self._config_fn
})
self._output_regex = r'.*For JobID \[(?P<rawId>\S+)\] output will be stored' + \
' in the dir (?P<output_dn>.*)$'
self._end_of_proxy_lifetime = None
self._set_proxy_lifetime()
#if self._use_delegate is False:
# self._submit_args_dict['-a'] = ' '
self._lock_filename = os.path.join(os.path.expanduser("~"),
".gcFileLock")
self._delegated_proxy_filename = None
self._delegated_proxy_lock = os.path.join(os.path.expanduser("~"),
".gcDelegatedProxyLock")
def __init__(self, config, name):
cancel_executor = CancelAndPurgeJobs(config, CREAMCancelJobs(config),
CREAMPurgeJobs(config))
GridWMS.__init__(
self,
config,
name,
submit_exec=resolve_install_path('glite-ce-job-submit'),
output_exec=resolve_install_path('glite-ce-job-output'),
check_executor=CREAMCheckJobs(config),
cancel_executor=ChunkedExecutor(config, 'cancel', cancel_executor))
self._delegate_exec = resolve_install_path('glite-ce-delegate-proxy')
self._use_delegate = config.get_bool('try delegate',
True,
on_change=None)
self._chunk_size = config.get_int('job chunk size', 10, on_change=None)
self._submit_args_dict.update({
'-r': self._ce,
'--config-vo': self._config_fn
})
self._output_regex = r'.*For JobID \[(?P<rawId>\S+)\] output will be stored' + \
' in the dir (?P<output_dn>.*)$'
if self._use_delegate is False:
self._submit_args_dict['-a'] = ' '
def __init__(self, config, name, check_executor=None):
GridWMS.__init__(
self,
config,
name,
submit_exec=resolve_install_path('glite-wms-job-submit'),
output_exec=resolve_install_path('glite-wms-job-output'),
check_executor=check_executor
or GridCheckJobs(config, 'glite-wms-job-status'),
cancel_executor=GridCancelJobs(config, 'glite-wms-job-cancel'))
self._delegate_exec = resolve_install_path(
'glite-wms-job-delegate-proxy')
self._submit_args_dict.update({
'-r': self._ce,
'--config': self._config_fn
})
self._use_delegate = config.get_bool('try delegate',
True,
on_change=None)
self._force_delegate = config.get_bool('force delegate',
False,
on_change=None)
self._discovery_plugin = None
if config.get_bool('discover wms', True, on_change=None):
self._discovery_plugin = DiscoverGliteEndpointsLazy(config)
self._discover_sites = config.get_bool('discover sites',
False,
on_change=None)
def __init__(self, config, name):
GridWMS.__init__(self, config, name,
submit_exec=resolve_install_path('edg-job-submit'),
output_exec=resolve_install_path('edg-job-get-output'),
check_executor=GridCheckJobs(config, 'edg-job-status'),
cancel_executor=GridCancelJobs(config, 'edg-job-cancel'),
jdl_writer=EDGJDL())
self._submit_args_dict.update({'-r': self._ce, '--config-vo': self._config_fn})
def __init__(self, config, name):
GridWMS.__init__(self, config, name)
self._submitExec = utils.resolveInstallPath('edg-job-submit')
self._statusExec = utils.resolveInstallPath('edg-job-status')
self._outputExec = utils.resolveInstallPath('edg-job-get-output')
self._cancelExec = utils.resolveInstallPath('edg-job-cancel')
self._submitParams.update({'-r': self._ce, '--config-vo': self._configVO })
def __init__(self, config, name):
deprecated('Please use the GliteWMS backend for grid jobs!')
GridWMS.__init__(self, config, name,
submit_exec=resolve_install_path('glite-job-submit'),
output_exec=resolve_install_path('glite-job-output'),
check_executor=GridCheckJobs(config, 'glite-job-status'),
cancel_executor=GridCancelJobs(config, 'glite-job-cancel'))
self._submit_args_dict.update({'-r': self._ce, '--config-vo': self._config_fn})
def __init__(self, config, name):
utils.deprecated('Please use the GliteWMS backend for grid jobs!')
GridWMS.__init__(self, config, name,
checkExecutor = Grid_CheckJobs(config, 'glite-job-status'),
cancelExecutor = Grid_CancelJobs(config, 'glite-job-cancel'))
self._submitExec = utils.resolveInstallPath('glite-job-submit')
self._outputExec = utils.resolveInstallPath('glite-job-output')
self._submitParams.update({'-r': self._ce, '--config-vo': self._configVO })
def __init__(self, config, name):
GridWMS.__init__(self, config, name,
checkExecutor = Grid_CheckJobs(config, 'edg-job-status'),
cancelExecutor = Grid_CancelJobs(config, 'edg-job-cancel'),
jdlWriter = EDGJDL())
self._submitExec = utils.resolveInstallPath('edg-job-submit')
self._outputExec = utils.resolveInstallPath('edg-job-get-output')
self._submitParams.update({'-r': self._ce, '--config-vo': self._configVO })
def __init__(self, config, name):
utils.deprecated("Please use the GliteWMS backend for grid jobs!")
GridWMS.__init__(self, config, name)
self._submitExec = utils.resolveInstallPath("glite-job-submit")
self._statusExec = utils.resolveInstallPath("glite-job-status")
self._outputExec = utils.resolveInstallPath("glite-job-output")
self._cancelExec = utils.resolveInstallPath("glite-job-cancel")
self._submitParams.update({"-r": self._ce, "--config-vo": self._configVO})
def __init__(self, config, name):
utils.deprecated('Please use the GliteWMS backend for grid jobs!')
GridWMS.__init__(self, config, name)
self._submitExec = utils.resolveInstallPath('glite-job-submit')
self._statusExec = utils.resolveInstallPath('glite-job-status')
self._outputExec = utils.resolveInstallPath('glite-job-output')
self._cancelExec = utils.resolveInstallPath('glite-job-cancel')
self._submitParams.update({'-r': self._ce, '--config-vo': self._configVO })
def __init__(self, config, name):
GridWMS.__init__(self, config, name)
self._submitExec = utils.resolveInstallPath('edg-job-submit')
self._statusExec = utils.resolveInstallPath('edg-job-status')
self._outputExec = utils.resolveInstallPath('edg-job-get-output')
self._cancelExec = utils.resolveInstallPath('edg-job-cancel')
self._submitParams.update({
'-r': self._ce,
'--config-vo': self._configVO
})
def __init__(self, config, name):
GridWMS.__init__(
self,
config,
name,
submit_exec=resolve_install_path('edg-job-submit'),
output_exec=resolve_install_path('edg-job-get-output'),
check_executor=GridCheckJobs(config, 'edg-job-status'),
cancel_executor=GridCancelJobs(config, 'edg-job-cancel'),
jdl_writer=EDGJDL())
self._submit_args_dict.update({
'-r': self._ce,
'--config-vo': self._config_fn
})
def __init__(self, config, name, check_executor=None):
GridWMS.__init__(self, config, name,
submit_exec=resolve_install_path('glite-wms-job-submit'),
output_exec=resolve_install_path('glite-wms-job-output'),
check_executor=check_executor or GridCheckJobs(config, 'glite-wms-job-status'),
cancel_executor=GridCancelJobs(config, 'glite-wms-job-cancel'))
self._delegate_exec = resolve_install_path('glite-wms-job-delegate-proxy')
self._submit_args_dict.update({'-r': self._ce, '--config': self._config_fn})
self._use_delegate = config.get_bool('try delegate', True, on_change=None)
self._force_delegate = config.get_bool('force delegate', False, on_change=None)
self._discovery_plugin = None
if config.get_bool('discover wms', True, on_change=None):
self._discovery_plugin = DiscoverGliteEndpointsLazy(config)
self._discover_sites = config.get_bool('discover sites', False, on_change=None)
def __init__(self, config, name):
deprecated('Please use the GliteWMS backend for grid jobs!')
GridWMS.__init__(self,
config,
name,
submit_exec=resolve_install_path('glite-job-submit'),
output_exec=resolve_install_path('glite-job-output'),
check_executor=GridCheckJobs(config,
'glite-job-status'),
cancel_executor=GridCancelJobs(
config, 'glite-job-cancel'))
self._submit_args_dict.update({
'-r': self._ce,
'--config-vo': self._config_fn
})
def __init__(self, config, name):
GridWMS.__init__(self, config, name)
self._delegateExec = utils.resolveInstallPath('glite-wms-job-delegate-proxy')
self._submitExec = utils.resolveInstallPath('glite-wms-job-submit')
self._statusExec = utils.resolveInstallPath('glite-wms-job-status')
self._outputExec = utils.resolveInstallPath('glite-wms-job-output')
self._cancelExec = utils.resolveInstallPath('glite-wms-job-cancel')
self._submitParams.update({'-r': self._ce, '--config': self._configVO})
self._useDelegate = config.getBool('try delegate', True, onChange = None)
self._forceDelegate = config.getBool('force delegate', False, onChange = None)
self._discovery_module = None
if config.getBool('discover wms', True, onChange = None):
self._discovery_module = DiscoverWMS_Lazy(config)
self._discover_sites = config.getBool('discover sites', False, onChange = None)
def __init__(self, config, name):
cancelExecutor = CancelAndPurgeJobs(config, CREAM_CancelJobs(config), CREAM_PurgeJobs(config))
GridWMS.__init__(self, config, name, checkExecutor = CREAM_CheckJobs(config),
cancelExecutor = ChunkedExecutor(config, 'cancel', cancelExecutor))
self._nJobsPerChunk = config.getInt('job chunk size', 10, onChange = None)
self._submitExec = utils.resolveInstallPath('glite-ce-job-submit')
self._outputExec = utils.resolveInstallPath('glite-ce-job-output')
self._submitParams.update({'-r': self._ce, '--config-vo': self._configVO })
self._outputRegex = r'.*For JobID \[(?P<rawId>\S+)\] output will be stored in the dir (?P<outputDir>.*)$'
self._useDelegate = False
if self._useDelegate is False:
self._submitParams.update({ '-a': ' ' })
def __init__(self, config, name):
GridWMS.__init__(self,
config,
name,
checkExecutor=Grid_CheckJobs(config,
'edg-job-status'),
cancelExecutor=Grid_CancelJobs(
config, 'edg-job-cancel'),
jdlWriter=EDGJDL())
self._submitExec = utils.resolveInstallPath('edg-job-submit')
self._outputExec = utils.resolveInstallPath('edg-job-get-output')
self._submitParams.update({
'-r': self._ce,
'--config-vo': self._configVO
})
def submit_jobs(self, jobnum_list, task):
if not self._begin_bulk_submission(): # Trying to delegate proxy failed
if self._force_delegate: # User switched on forcing delegation => exception
raise BackendError('Unable to delegate proxy!')
self._log.error('Unable to delegate proxy! Continue with automatic delegation...')
self._submit_args_dict.update({'-a': ' '})
self._use_delegate = False
for result in GridWMS.submit_jobs(self, jobnum_list, task):
yield result
def submitJobs(self, jobNumList, module):
if not self.bulkSubmissionBegin(): # Trying to delegate proxy failed
if self._forceDelegate: # User switched on forcing delegation => exception
raise BackendError('Unable to delegate proxy!')
utils.eprint('Unable to delegate proxy! Continue with automatic delegation...')
self._submitParams.update({ '-a': ' ' })
self._useDelegate = False
for submitInfo in GridWMS.submitJobs(self, jobNumList, module):
yield submitInfo
def submit_jobs(self, jobnum_list, task):
if not self._begin_bulk_submission(
): # Trying to delegate proxy failed
self._log.error(
'Unable to delegate proxy! Continue with automatic delegation...'
)
self._submit_args_dict.update({'-a': ' '})
self._use_delegate = False
for result in GridWMS.submit_jobs(self, jobnum_list, task):
yield result
def submit_jobs(self, jobnum_list, task):
if not self._begin_bulk_submission(
): # Trying to delegate proxy failed
if self._force_delegate: # User switched on forcing delegation => exception
raise BackendError('Unable to delegate proxy!')
self._log.error(
'Unable to delegate proxy! Continue with automatic delegation...'
)
self._submit_args_dict.update({'-a': ' '})
self._use_delegate = False
for result in GridWMS.submit_jobs(self, jobnum_list, task):
yield result
def __init__(self, config, name):
GridWMS.__init__(self, config, name)
self._nJobsPerChunk = config.getInt('job chunk size', 10, onChange = None)
self._submitExec = utils.resolveInstallPath('glite-ce-job-submit')
self._statusExec = utils.resolveInstallPath('glite-ce-job-status')
self._outputExec = utils.resolveInstallPath('glite-ce-job-output')
self._cancelExec = utils.resolveInstallPath('glite-ce-job-cancel')
self._purgeExec = utils.resolveInstallPath('glite-ce-job-purge')
self._submitParams.update({'-r': self._ce, '--config-vo': self._configVO })
lvl0_status_ok = r'.*JobID=\[(?P<rawId>\S+)\]\s+Status\s+=\s+\[(?P<status>\S+)\].*'
lvl0_status_err = r'.*JobID=\[(?P<rawId>\S+)\]\s+For this job CREAM has returned a fault: MethodName=\[(?P<methodName>.*)\] '
lvl0_status_err += r'Timestamp=\[(?P<timestamp>.*)\] ErrorCode=\[(?P<errorCode>.*)\] '
lvl0_status_err += r'Description=\[(?P<description>.*)\] FaultCause=\[(?P<faultCause>.*)\].*'
self._statusRegexLevel0 = [lvl0_status_ok, lvl0_status_err]
self._outputRegex = r'.*For JobID \[(?P<rawId>\S+)\] output will be stored in the dir (?P<outputDir>.*)$'
self._useDelegate = False
if self._useDelegate is False:
self._submitParams.update({ '-a': ' ' })
def submit_jobs(self, jobnum_list, task):
import os
activity = Activity("Waiting for lock to be released...")
while os.path.isfile(self._lock_filename):
time.sleep(2)
file = open(self._lock_filename, "w+")
activity.finish()
activity = Activity("Lock acquired:" + self._lock_filename)
activity.finish()
t = self._begin_bulk_submission()
while not t:
activity = Activity(
'waiting before trying to delegate proxy again...')
time.sleep(900)
activity.finish()
activity = Activity('re-attempting to delegate proxy...')
t = self._begin_bulk_submission()
activity.finish()
'''
if not self._begin_bulk_submission(): # Trying to delegate proxy failed
self._log.error('Unable to delegate proxy! Continue with automatic delegation...')
self._submit_args_dict.update({'-a': ' '})
self._use_delegate = False
'''
count_submitted = 0
for result in GridWMS.submit_jobs(self, jobnum_list, task):
count_submitted += 1
yield result
file.close()
self._log.info('count_submitted: %d' % count_submitted)
count_submitted = int(count_submitted * 0.2)
x = threading.Thread(target=self.delfile,
args=(self._lock_filename, count_submitted,
self._log))
x.start()
def _make_jdl(self, jobnum, task):
return ['[\n'] + GridWMS._make_jdl(self, jobnum, task) + [
'OutputSandboxBaseDestUri = "gsiftp://localhost";\n]'
]
def makeJDL(self, jobNum, module): return ['[\n'] + GridWMS.makeJDL(self, jobNum, module) + ['OutputSandboxBaseDestUri = "gsiftp://localhost";\n]']
