def _iter_blocks_raw(self): def _filter_block(block): if self._filter: return self._filter in '/%s#' % DataProvider.get_block_id(block) return True try: fp = SafeFile(self._filename) except Exception: raise DatasetError('Unable to open dataset file %s' % repr(self._filename)) for block in self._create_blocks(fp.iter_close()): if _filter_block(block): self._raise_on_abort() yield block
def __init__(self, config, name, task): LocalEventHandler.__init__(self, config, name, task) self._source_jid = config.get('source jid', on_change=None) self._target_jid = config.get('target jid', on_change=None) password_fn = config.get_fn('source password file') os.chmod(password_fn, stat.S_IRUSR) # password in variable name removes it from debug log! self._source_password = SafeFile(password_fn).read_close().strip() try: # xmpp contains many deprecated constructs import warnings warnings.simplefilter('ignore', DeprecationWarning) except Exception: clear_current_exception() self._xmpp = ignore_exception(Exception, None, __import__, 'xmpp') if self._xmpp is None: try: import grid_control_gui.xmpp self._xmpp = grid_control_gui.xmpp except Exception: raise Exception('Unable to load jabber library!')
def _submit_job(self, jobnum, task): # Submit job and yield (jobnum, WMS ID, other data) jdl_fd, jdl_fn = tempfile.mkstemp('.jdl') try: jdl_line_list = self._make_jdl(jobnum, task) safe_write(os.fdopen(jdl_fd, 'w'), jdl_line_list) except Exception: remove_files([jdl_fn]) raise BackendError('Could not write jdl data to %s.' % jdl_fn) try: submit_arg_list = [] for key_value in filter_dict(self._submit_args_dict, value_filter=identity).items(): submit_arg_list.extend(key_value) submit_arg_list.append(jdl_fn) activity = Activity('submitting job %d' % jobnum) proc = LocalProcess(self._submit_exec, '--nomsg', '--noint', '--logfile', '/dev/stderr', *submit_arg_list) wms_id = None stripped_stdout_iter = imap(str.strip, proc.stdout.iter(timeout=60)) for line in ifilter(lambda x: x.startswith('http'), stripped_stdout_iter): wms_id = line exit_code = proc.status(timeout=0, terminate=True) activity.finish() if (exit_code != 0) or (wms_id is None): if self._explain_error(proc, exit_code): pass else: self._log.log_process( proc, files={'jdl': SafeFile(jdl_fn).read()}) finally: remove_files([jdl_fn]) job_data = {'jdl': str.join('', jdl_line_list)} return (jobnum, self._create_gc_id(wms_id), job_data)
def gc_run(args=None, intro=True): # display the 'grid-control' logo and version if intro and not os.environ.get('GC_DISABLE_INTRO'): sys.stdout.write( SafeFile(get_path_share('logo.txt'), 'r').read_close()) sys.stdout.write('Revision: %s\n' % get_version()) pyver = (sys.version_info[0], sys.version_info[1]) if pyver < (2, 3): deprecated('This python version (%d.%d) is not supported anymore!' % pyver) atexit.register(lambda: sys.stdout.write('\n')) # main try... except block to catch exceptions and show error message try: return _gc_run(args) except SystemExit: # avoid getting caught for Python < 2.5 abort(True) raise except Exception: # coverage overrides sys.excepthook abort(True) gc_excepthook(*sys.exc_info()) sys.exit(os.EX_SOFTWARE)
def _log_tar(only_print=False): # self._log.info('tar: %s' % self._fn) sleep_when_cannot_accept_jobs = False message = "" if not only_print: tar = tarfile.TarFile.open(self._fn, 'a') for key, value in record.files.items(): if "The CREAM service cannot accept jobs at the moment" in value: sleep_when_cannot_accept_jobs = True message = "The CREAM service cannot accept jobs at the moment" elif "Unable to connect to" in value: sleep_when_cannot_accept_jobs = True message = value value = os.linesep.join([s for s in value.splitlines() if s]) if only_print: self._log.info('\n\tkey: "%s"\n\tvalue: "%s"' % (key, value)) else: if value.startswith('\n'): value = value[1:] if os.path.exists(value): value = SafeFile(value).read_close() # self._log.info('\tvirtual file: "%s"' % os.path.join(entry, key)) file_obj = VirtualFile(os.path.join(entry, key), [value]) info, handle = file_obj.get_tar_info() # self._log.info('\tinfo: "%s"' % info) # self._log.info('\thandle: "%s"' % handle) tar.addfile(info, handle) handle.close() if not only_print: tar.close() if sleep_when_cannot_accept_jobs: from grid_control.utils.activity import Activity activity = Activity( message + '. Waiting before trying to delegate proxy again...') time.sleep(900) activity.finish()
def __init__(self, config, name, check_executor, cancel_executor): WMS.__init__(self, config, name) for executor in [check_executor, cancel_executor]: executor.setup(self._log) (self._check_executor, self._cancel_executor) = (check_executor, cancel_executor) if self._name != self.__class__.__name__.upper(): self._log.info('Using batch system: %s (%s)', self.__class__.__name__, self._name) else: self._log.info('Using batch system: %s', self._name) self._runlib = config.get_work_path('gc-run.lib') fp = SafeFile(self._runlib, 'w') content = SafeFile(get_path_share('gc-run.lib')).read() fp.write(content.replace('__GC_VERSION__', __import__('grid_control').__version__)) fp.close() self._path_output = config.get_work_path('output') self._path_file_cache = config.get_work_path('files') ensure_dir_exists(self._path_output, 'output directory') self._path_fail = config.get_work_path('fail') # Initialise access token and storage managers # UI -> SE -> WN self._sm_se_in = config.get_plugin('se input manager', 'SEStorageManager', cls=StorageManager, bind_kwargs={'tags': [self]}, pargs=('se', 'se input', 'SE_INPUT')) self._sm_sb_in = config.get_plugin('sb input manager', 'LocalSBStorageManager', cls=StorageManager, bind_kwargs={'tags': [self]}, pargs=('sandbox', 'sandbox', 'SB_INPUT')) # UI <- SE <- WN self._sm_se_out = config.get_plugin('se output manager', 'SEStorageManager', cls=StorageManager, bind_kwargs={'tags': [self]}, pargs=('se', 'se output', 'SE_OUTPUT')) self._sm_sb_out = None self._token = config.get_composited_plugin(['proxy', 'access token'], 'TrivialAccessToken', 'MultiAccessToken', cls=AccessToken, bind_kwargs={'inherit': True, 'tags': [self]}) self._output_fn_list = None
def _config_is_instrumented(self, fn): cfg = SafeFile(fn).read_close() for tag in self._needed_vn_set: if (not '__%s__' % tag in cfg) and (not '@%s@' % tag in cfg): return False return True
def _check_write_stack_log(): if os.path.exists('gc_debug_stack.log'): with_file( SafeFile('gc_debug_stack.log', 'w'), lambda fp: DebugInterface( stream=fp).show_stack(thread_id='all'))
def deprecated(text): log = logging.getLogger('console') log.critical('\n%s\n[DEPRECATED] %s', SafeFile(get_path_share('fail.txt')).read_close(), text) if not UserInputInterface().prompt_bool('Do you want to continue?', False): sys.exit(os.EX_TEMPFAIL)
def save_to_disk(self, filename): with_file(SafeFile(filename, 'w'), partial(pickle.dump, self))
def commit(self, jobnum, job_obj): with_file( SafeFile(os.path.join(self._path_db, 'job_%d.txt' % jobnum), 'w'), lambda fp: fp.writelines( self._fmt.format(self._serialize_job_obj(job_obj)))) self._job_map[jobnum] = job_obj
def dump_dbs3_json(dn, block_dump_iter): for block_dump in block_dump_iter: block_dump_fn = block_dump['block']['block_name'].strip('/').replace( '/', '_') + '.json' with_file(SafeFile(os.path.join(dn, block_dump_fn), 'w'), partial(json.dump, block_dump))
def _show_image(self, name, buffer): with_file(SafeFile(name, 'wb'), lambda fp: fp.write(buffer.getvalue()))
def _parse_cmd_line(cmd_line_args): # grid-control command line parser parser = Options(usage='%s [OPTIONS] <config file>', add_help_option=False) parser.add_bool(None, ' ', 'debug', default=False) parser.add_bool(None, ' ', 'help-conf', default=False) parser.add_bool(None, ' ', 'help-confmin', default=False) parser.add_bool(None, 'c', 'continuous', default=False) parser.add_bool(None, 'h', 'help', default=False) parser.add_bool(None, 'i', 'init', default=False) parser.add_bool(None, 'q', 'resync', default=False) parser.add_bool(None, 's', 'no-submission', default=True, dest='submission') parser.add_bool(None, 'G', 'gui', default=False, dest='gui_ansi') parser.add_bool(None, ' ', 'createwd', default=False) parser.add_bool(None, ' ', 'forcedeljob', default=False) parser.add_accu(None, 'v', 'verbose') parser.add_list(None, 'l', 'logging') parser.add_list(None, 'o', 'override') parser.add_text(None, 'a', 'action') parser.add_text(None, 'd', 'delete') parser.add_text(None, 'C', 'cancel') parser.add_text(None, 'J', 'job-selector') parser.add_text(None, 'n', 'jobs') parser.add_text(None, 'm', 'max-retry') parser.add_text(None, ' ', 'reset') parser.add_bool(None, ' ', 'debug-console', False) # undocumented debug option parser.add_list(None, ' ', 'debug-trace') # undocumented debug option # Deprecated options - refer to new report script instead for (sopt, lopt) in [('-r', 'report'), ('-R', 'site-report'), ('-T', 'time-report'), ('-M', 'task-report'), ('-D', 'detail-report'), ('', 'help-vars')]: parser.add_bool(None, sopt, lopt, default=False, dest='old_report') (opts, args, _) = parser.parse(args=cmd_line_args) opts.gui = None if opts.gui_ansi: opts.gui = 'ANSIGUI' opts.continuous = opts.continuous or None # either True or None # Display help if opts.help: parser.exit_with_usage(msg=SafeFile( get_path_share('help.txt')).read_close(), show_help=False) # Require single config file argument if len(args) == 0: parser.exit_with_usage(msg='Config file not specified!') elif len(args) > 1: parser.exit_with_usage(msg='Invalid command line arguments: %r' % cmd_line_args) # Warn about deprecated report options if opts.old_report: deprecated( 'Please use the more versatile report tool in the scripts directory!' ) # Configure preliminary logging logging.getLogger().setLevel(max(1, logging.DEFAULT - opts.verbose)) return (opts, args)
def _config_store_backup(self, source, target, fragment_path=None): content = SafeFile(source).read_close() if fragment_path: self._log.info('Instrumenting... %s', os.path.basename(source)) content += SafeFile(fragment_path).read_close() SafeFile(target, 'w').write_close(content)
def read_from_disk(filename): return with_file(SafeFile(filename), pickle.load)
def _write_file(self, fn, msg=None, **kwargs): def _write_msg_view(fp): if msg is not None: fp.write(msg) self._view.write(fp, **kwargs) with_file(SafeFile(fn, 'w'), _write_msg_view)