def _iter_blocks_raw(self):
def _filter_block(block):
if self._filter:
return self._filter in '/%s#' % DataProvider.get_block_id(block)
return True
try:
fp = SafeFile(self._filename)
except Exception:
raise DatasetError('Unable to open dataset file %s' % repr(self._filename))
for block in self._create_blocks(fp.iter_close()):
if _filter_block(block):
self._raise_on_abort()
yield block
def __init__(self, config, name, task):
LocalEventHandler.__init__(self, config, name, task)
self._source_jid = config.get('source jid', on_change=None)
self._target_jid = config.get('target jid', on_change=None)
password_fn = config.get_fn('source password file')
os.chmod(password_fn, stat.S_IRUSR)
# password in variable name removes it from debug log!
self._source_password = SafeFile(password_fn).read_close().strip()
try: # xmpp contains many deprecated constructs
import warnings
warnings.simplefilter('ignore', DeprecationWarning)
except Exception:
clear_current_exception()
self._xmpp = ignore_exception(Exception, None, __import__, 'xmpp')
if self._xmpp is None:
try:
import grid_control_gui.xmpp
self._xmpp = grid_control_gui.xmpp
except Exception:
raise Exception('Unable to load jabber library!')
def _submit_job(self, jobnum, task):
# Submit job and yield (jobnum, WMS ID, other data)
jdl_fd, jdl_fn = tempfile.mkstemp('.jdl')
try:
jdl_line_list = self._make_jdl(jobnum, task)
safe_write(os.fdopen(jdl_fd, 'w'), jdl_line_list)
except Exception:
remove_files([jdl_fn])
raise BackendError('Could not write jdl data to %s.' % jdl_fn)
try:
submit_arg_list = []
for key_value in filter_dict(self._submit_args_dict,
value_filter=identity).items():
submit_arg_list.extend(key_value)
submit_arg_list.append(jdl_fn)
activity = Activity('submitting job %d' % jobnum)
proc = LocalProcess(self._submit_exec, '--nomsg', '--noint',
'--logfile', '/dev/stderr', *submit_arg_list)
wms_id = None
stripped_stdout_iter = imap(str.strip,
proc.stdout.iter(timeout=60))
for line in ifilter(lambda x: x.startswith('http'),
stripped_stdout_iter):
wms_id = line
exit_code = proc.status(timeout=0, terminate=True)
activity.finish()
if (exit_code != 0) or (wms_id is None):
if self._explain_error(proc, exit_code):
pass
else:
self._log.log_process(
proc, files={'jdl': SafeFile(jdl_fn).read()})
finally:
remove_files([jdl_fn])
job_data = {'jdl': str.join('', jdl_line_list)}
return (jobnum, self._create_gc_id(wms_id), job_data)
def gc_run(args=None, intro=True):
# display the 'grid-control' logo and version
if intro and not os.environ.get('GC_DISABLE_INTRO'):
sys.stdout.write(
SafeFile(get_path_share('logo.txt'), 'r').read_close())
sys.stdout.write('Revision: %s\n' % get_version())
pyver = (sys.version_info[0], sys.version_info[1])
if pyver < (2, 3):
deprecated('This python version (%d.%d) is not supported anymore!' %
pyver)
atexit.register(lambda: sys.stdout.write('\n'))
# main try... except block to catch exceptions and show error message
try:
return _gc_run(args)
except SystemExit: # avoid getting caught for Python < 2.5
abort(True)
raise
except Exception: # coverage overrides sys.excepthook
abort(True)
gc_excepthook(*sys.exc_info())
sys.exit(os.EX_SOFTWARE)
def _log_tar(only_print=False):
# self._log.info('tar: %s' % self._fn)
sleep_when_cannot_accept_jobs = False
message = ""
if not only_print:
tar = tarfile.TarFile.open(self._fn, 'a')
for key, value in record.files.items():
if "The CREAM service cannot accept jobs at the moment" in value:
sleep_when_cannot_accept_jobs = True
message = "The CREAM service cannot accept jobs at the moment"
elif "Unable to connect to" in value:
sleep_when_cannot_accept_jobs = True
message = value
value = os.linesep.join([s for s in value.splitlines() if s])
if only_print:
self._log.info('\n\tkey: "%s"\n\tvalue: "%s"' %
(key, value))
else:
if value.startswith('\n'):
value = value[1:]
if os.path.exists(value):
value = SafeFile(value).read_close()
# self._log.info('\tvirtual file: "%s"' % os.path.join(entry, key))
file_obj = VirtualFile(os.path.join(entry, key), [value])
info, handle = file_obj.get_tar_info()
# self._log.info('\tinfo: "%s"' % info)
# self._log.info('\thandle: "%s"' % handle)
tar.addfile(info, handle)
handle.close()
if not only_print:
tar.close()
if sleep_when_cannot_accept_jobs:
from grid_control.utils.activity import Activity
activity = Activity(
message +
'. Waiting before trying to delegate proxy again...')
time.sleep(900)
activity.finish()
def __init__(self, config, name, check_executor, cancel_executor):
WMS.__init__(self, config, name)
for executor in [check_executor, cancel_executor]:
executor.setup(self._log)
(self._check_executor, self._cancel_executor) = (check_executor, cancel_executor)
if self._name != self.__class__.__name__.upper():
self._log.info('Using batch system: %s (%s)', self.__class__.__name__, self._name)
else:
self._log.info('Using batch system: %s', self._name)
self._runlib = config.get_work_path('gc-run.lib')
fp = SafeFile(self._runlib, 'w')
content = SafeFile(get_path_share('gc-run.lib')).read()
fp.write(content.replace('__GC_VERSION__', __import__('grid_control').__version__))
fp.close()
self._path_output = config.get_work_path('output')
self._path_file_cache = config.get_work_path('files')
ensure_dir_exists(self._path_output, 'output directory')
self._path_fail = config.get_work_path('fail')
# Initialise access token and storage managers
# UI -> SE -> WN
self._sm_se_in = config.get_plugin('se input manager', 'SEStorageManager',
cls=StorageManager, bind_kwargs={'tags': [self]}, pargs=('se', 'se input', 'SE_INPUT'))
self._sm_sb_in = config.get_plugin('sb input manager', 'LocalSBStorageManager',
cls=StorageManager, bind_kwargs={'tags': [self]}, pargs=('sandbox', 'sandbox', 'SB_INPUT'))
# UI <- SE <- WN
self._sm_se_out = config.get_plugin('se output manager', 'SEStorageManager',
cls=StorageManager, bind_kwargs={'tags': [self]}, pargs=('se', 'se output', 'SE_OUTPUT'))
self._sm_sb_out = None
self._token = config.get_composited_plugin(['proxy', 'access token'], 'TrivialAccessToken',
'MultiAccessToken', cls=AccessToken, bind_kwargs={'inherit': True, 'tags': [self]})
self._output_fn_list = None
def _config_is_instrumented(self, fn):
cfg = SafeFile(fn).read_close()
for tag in self._needed_vn_set:
if (not '__%s__' % tag in cfg) and (not '@%s@' % tag in cfg):
return False
return True
def _check_write_stack_log():
if os.path.exists('gc_debug_stack.log'):
with_file(
SafeFile('gc_debug_stack.log', 'w'), lambda fp: DebugInterface(
stream=fp).show_stack(thread_id='all'))
def deprecated(text):
log = logging.getLogger('console')
log.critical('\n%s\n[DEPRECATED] %s',
SafeFile(get_path_share('fail.txt')).read_close(), text)
if not UserInputInterface().prompt_bool('Do you want to continue?', False):
sys.exit(os.EX_TEMPFAIL)
def save_to_disk(self, filename):
with_file(SafeFile(filename, 'w'), partial(pickle.dump, self))
def commit(self, jobnum, job_obj):
with_file(
SafeFile(os.path.join(self._path_db, 'job_%d.txt' % jobnum), 'w'),
lambda fp: fp.writelines(
self._fmt.format(self._serialize_job_obj(job_obj))))
self._job_map[jobnum] = job_obj
def dump_dbs3_json(dn, block_dump_iter):
for block_dump in block_dump_iter:
block_dump_fn = block_dump['block']['block_name'].strip('/').replace(
'/', '_') + '.json'
with_file(SafeFile(os.path.join(dn, block_dump_fn), 'w'),
partial(json.dump, block_dump))
def _show_image(self, name, buffer):
with_file(SafeFile(name, 'wb'), lambda fp: fp.write(buffer.getvalue()))
def _parse_cmd_line(cmd_line_args):
# grid-control command line parser
parser = Options(usage='%s [OPTIONS] <config file>', add_help_option=False)
parser.add_bool(None, ' ', 'debug', default=False)
parser.add_bool(None, ' ', 'help-conf', default=False)
parser.add_bool(None, ' ', 'help-confmin', default=False)
parser.add_bool(None, 'c', 'continuous', default=False)
parser.add_bool(None, 'h', 'help', default=False)
parser.add_bool(None, 'i', 'init', default=False)
parser.add_bool(None, 'q', 'resync', default=False)
parser.add_bool(None,
's',
'no-submission',
default=True,
dest='submission')
parser.add_bool(None, 'G', 'gui', default=False, dest='gui_ansi')
parser.add_bool(None, ' ', 'createwd', default=False)
parser.add_bool(None, ' ', 'forcedeljob', default=False)
parser.add_accu(None, 'v', 'verbose')
parser.add_list(None, 'l', 'logging')
parser.add_list(None, 'o', 'override')
parser.add_text(None, 'a', 'action')
parser.add_text(None, 'd', 'delete')
parser.add_text(None, 'C', 'cancel')
parser.add_text(None, 'J', 'job-selector')
parser.add_text(None, 'n', 'jobs')
parser.add_text(None, 'm', 'max-retry')
parser.add_text(None, ' ', 'reset')
parser.add_bool(None, ' ', 'debug-console',
False) # undocumented debug option
parser.add_list(None, ' ', 'debug-trace') # undocumented debug option
# Deprecated options - refer to new report script instead
for (sopt, lopt) in [('-r', 'report'), ('-R', 'site-report'),
('-T', 'time-report'), ('-M', 'task-report'),
('-D', 'detail-report'), ('', 'help-vars')]:
parser.add_bool(None, sopt, lopt, default=False, dest='old_report')
(opts, args, _) = parser.parse(args=cmd_line_args)
opts.gui = None
if opts.gui_ansi:
opts.gui = 'ANSIGUI'
opts.continuous = opts.continuous or None # either True or None
# Display help
if opts.help:
parser.exit_with_usage(msg=SafeFile(
get_path_share('help.txt')).read_close(),
show_help=False)
# Require single config file argument
if len(args) == 0:
parser.exit_with_usage(msg='Config file not specified!')
elif len(args) > 1:
parser.exit_with_usage(msg='Invalid command line arguments: %r' %
cmd_line_args)
# Warn about deprecated report options
if opts.old_report:
deprecated(
'Please use the more versatile report tool in the scripts directory!'
)
# Configure preliminary logging
logging.getLogger().setLevel(max(1, logging.DEFAULT - opts.verbose))
return (opts, args)
def _config_store_backup(self, source, target, fragment_path=None):
content = SafeFile(source).read_close()
if fragment_path:
self._log.info('Instrumenting... %s', os.path.basename(source))
content += SafeFile(fragment_path).read_close()
SafeFile(target, 'w').write_close(content)
def read_from_disk(filename):
return with_file(SafeFile(filename), pickle.load)
def _write_file(self, fn, msg=None, **kwargs): def _write_msg_view(fp): if msg is not None: fp.write(msg) self._view.write(fp, **kwargs) with_file(SafeFile(fn, 'w'), _write_msg_view)