def job_submit(rsl, args, proxy=None): cmd = [GLOBUSRUN_WS, ] cmd.extend(args) rslfile = mktemp() fd = open(rslfile, "w") fd.write(rsl) fd.close() cmd.extend(['-f', rslfile]) proxy_filename = None env = dict(os.environ) if proxy: proxy_filename = prepare_proxy(proxy) env['X509_USER_PROXY'] = proxy_filename pid = Popen(cmd, stdout=PIPE, stderr=PIPE, env=env, close_fds=True) (epr, err) = tools.communicate_with_timeout(pid, timeout=pilot.spooler.config['job_submit_timeout']) if proxy_filename: os.unlink(proxy_filename) os.unlink(rslfile) if epr is None: if pid.poll() is None: os.kill(pid.pid, signal.SIGKILL) raise TimeoutError('globusrun-ws timed out after %s seconds' % pilot.spooler.config['job_submit_timeout']) if pid.returncode != 0: raise GlobusError(err) return epr
def destroy(self): """ Clean up the resources used for this shell. """ for delegation_token_file in self._delegation_token_files: try: os.unlink(delegation_token_file) except: LOG.warning("Could not remove delegation token file %s" % delegation_token_file) try: self._delegation_token_files = None self._write_buffer.close() self._read_buffer.close() os.close(self._fd) os.close(self._child_fd) try: LOG.debug("Sending SIGKILL to process with PID %d" % (self.pid,)) os.kill(self.pid, signal.SIGKILL) _, exitcode = os.waitpid(self.pid, 0) msg = "%s - shell_id:%s pid:%d - Exited with status %d" % (self.username, self.shell_id, self.pid, exitcode) except OSError: msg = "%s - shell_id:%s pid:%d - Killed successfully" % (self.username, self.shell_id, self.pid,) # This means the subprocess was already killed, which happens if the command was "quit" # This can also happen if the waitpid call results in an error, which we don't care about. LOG.info(msg) SHELL_OUTPUT_LOGGER.info(msg) SHELL_INPUT_LOGGER.info(msg) finally: self.destroyed = True
def _writeContent(filepath, content): # Unlink before overwrite to cope with Backup/Restore restoring as root try: os.unlink(filepath) except OSError, e: # swallow "no such file", re-raise anything else if e.errno != errno.ENOENT: raise e
def job_kill(epr, proxy=None): cmd = [GLOBUSRUN_WS, '-kill', '-host-authz', '-quiet', '-job-epr-file'] eprfile = mktemp() fd = open(eprfile, "w") fd.write(epr) fd.close() cmd.append(eprfile) proxy_filename = None env = dict(os.environ) if proxy: proxy_filename = prepare_proxy(proxy) env['X509_USER_PROXY'] = proxy_filename pid = Popen(cmd, stdout=PIPE, stderr=PIPE, env=env, close_fds=True) (_, err) = tools.communicate_with_timeout(pid, timeout=pilot.spooler.config['job_kill_timeout']) if proxy_filename: os.unlink(proxy_filename) os.unlink(eprfile) if pid.poll() is None: os.kill(pid.pid, signal.SIGKILL) raise TimeoutError('globusrun-ws timed out after %s seconds' % pilot.spooler.config['job_kill_timeout']) if pid.returncode != 0: raise GlobusError("RC: %d, %s" % (pid.returncode, err))
def deleteContent(self): # The default implementation deletes all files under the TFTP directory # with a name containing a lowercase or uppercase MAC address that # matches the one assigned for the endpoint lcasemac = self._mac.replace(':', '').lower() ucasemac = self._mac.replace(':', '').upper() for filename in os.listdir(TFTP_DIR): if (lcasemac in filename) or (ucasemac in filename): os.unlink(TFTP_DIR + '/' + filename) # Unregister accounts self._unregister()
def job_status(epr, proxy=None): cmd = [WSRF_QUERY, ] eprfile = mktemp() cmd.extend(['-e', eprfile]) fd = open(eprfile, "w") fd.write(epr) fd.close() proxy_filename = None env = dict(os.environ) if proxy: proxy_filename = prepare_proxy(proxy) env['X509_USER_PROXY'] = proxy_filename pid = Popen(cmd, stdout=PIPE, stderr=PIPE, env=env, close_fds=True) (xml, err) = tools.communicate_with_timeout(pid, timeout=pilot.spooler.config['job_query_timeout']) if proxy_filename: os.unlink(proxy_filename) os.unlink(eprfile) if xml is None: if pid.poll() is None: os.kill(pid.pid, signal.SIGKILL) raise TimeoutError('wsrf-query timed out after %s seconds' % pilot.spooler.config['job_query_timeout']) if pid.returncode == 2: raise ProxyExpiredError(err) elif pid.returncode != 0: raise GlobusError("Exit Code %d, %s" % (pid.returncode, err)) gstate = State(etree.fromstring(xml)) if gstate.state is None: raise GlobusError("state element not present in wsrf-query response") return gstate
def __init__(self, shell_command, subprocess_env, shell_id, username, delegation_token_dir): try: user_info = pwd.getpwnam(username) except KeyError: LOG.error("Unix user account didn't exist at subprocess creation. Was it deleted?") raise parent, child = pty.openpty() try: tty.setraw(parent) except tty.error: LOG.debug("Could not set parent fd to raw mode, user will see duplicated input.") subprocess_env[constants.HOME] = user_info.pw_dir command_to_use = [_SETUID_PROG, str(user_info.pw_uid), str(user_info.pw_gid)] command_to_use.extend(shell_command) delegation_token_files = self._get_delegation_tokens(username, delegation_token_dir) if delegation_token_files: merged_token_file_path = self._merge_delegation_tokens(delegation_token_files, delegation_token_dir) for path in delegation_token_files: try: os.unlink(path) except: LOG.warning("Could not remove delegation token file %s" % path) delegation_token_files = [merged_token_file_path] subprocess_env[constants.HADOOP_TOKEN_FILE_LOCATION] = merged_token_file_path try: LOG.debug("Starting subprocess with command '%s' and environment '%s'" % (command_to_use, subprocess_env,)) p = subprocess.Popen(command_to_use, stdin=child, stdout=child, stderr=child, env=subprocess_env, close_fds=True) except (OSError, ValueError): os.close(parent) os.close(child) raise msg_format = "%s - shell_id:%s pid:%d - args:%s" msg_args = (username, shell_id, p.pid, ' '.join(command_to_use)) msg = msg_format % msg_args SHELL_OUTPUT_LOGGER.info(msg) SHELL_INPUT_LOGGER.info(msg) # State that shouldn't be touched by any other classes. self._output_buffer_length = 0 self._commands = [] self._fd = parent self._child_fd = child self.subprocess = p self.pid = p.pid self._write_buffer = cStringIO.StringIO() self._read_buffer = cStringIO.StringIO() self._delegation_token_files = delegation_token_files # State that's accessed by other classes. self.shell_id = shell_id self.username = username # Timestamp that is updated on shell creation and on every output request. Used so that we know # when to kill the shell. self.time_received = time.time() self.last_output_sent = False self.remove_at_next_iteration = False self.destroyed = False
def __init__(self, shell_command, subprocess_env, shell_id, username, delegation_token_dir): try: user_info = pwd.getpwnam(username) except KeyError: LOG.error("Unix user account didn't exist at subprocess creation. Was it deleted?") raise parent, child = pty.openpty() try: tty.setraw(parent) except tty.error: LOG.debug("Could not set parent fd to raw mode, user will see duplicated input.") subprocess_env[constants.HOME] = str(user_info.pw_dir) command_to_use = [str(_SETUID_PROG), str(user_info.pw_uid), str(user_info.pw_gid)] command_to_use.extend(shell_command) delegation_token_files = self._get_delegation_tokens(username, delegation_token_dir) if delegation_token_files: merged_token_file_path = self._merge_delegation_tokens(delegation_token_files, delegation_token_dir) for path in delegation_token_files: try: os.unlink(path) except: LOG.warning("Could not remove delegation token file %s" % path) delegation_token_files = [merged_token_file_path] subprocess_env[constants.HADOOP_TOKEN_FILE_LOCATION] = merged_token_file_path try: LOG.debug("Starting subprocess with command '%s' and environment '%s'" % (command_to_use, subprocess_env,)) p = subprocess.Popen(command_to_use, stdin=child, stdout=child, stderr=child, env=subprocess_env, close_fds=True) except (OSError, ValueError): os.close(parent) os.close(child) raise msg_format = "%s - shell_id:%s pid:%d - args:%s" msg_args = (username, shell_id, p.pid, ' '.join(command_to_use)) msg = msg_format % msg_args SHELL_OUTPUT_LOGGER.info(msg) SHELL_INPUT_LOGGER.info(msg) # State that shouldn't be touched by any other classes. self._output_buffer_length = 0 self._commands = [] self._fd = parent self._child_fd = child self.subprocess = p self.pid = p.pid self._write_buffer = cStringIO.StringIO() self._read_buffer = cStringIO.StringIO() self._delegation_token_files = delegation_token_files # State that's accessed by other classes. self.shell_id = shell_id self.username = username # Timestamp that is updated on shell creation and on every output request. Used so that we know # when to kill the shell. self.time_received = time.time() self.last_output_sent = False self.remove_at_next_iteration = False self.destroyed = False
def _load_bundle(bundle_desc): # pprint(bundle_desc) print "register bundle `%s'" % bundle_desc['name'] _bundle = store.find(Bundle, Bundle.name == bundle_desc['name']).one() or Bundle() _bundle.name = bundle_desc['name'] _bundle.description = bundle_desc['description'] store.add(_bundle) if 'options' in bundle_desc: for _option_desc in bundle_desc['options']: _option = store.find(Option, Option.name == _option_desc['name'], Option.bundle == bundle_desc['name']).one() or Option() _option.name = _option_desc['name'] _option.bundle = bundle_desc['name'] _option.description = _option_desc.get('description', _option_desc['name']) _option.value = unicode(_option_desc['value']) store.add(_option) if 'service' in bundle_desc: _service_desc = bundle_desc['service'] _service = store.find(Service, Service.name == bundle_desc['name']).one() or Service() _service.name = bundle_desc['name'] _service.bundle = bundle_desc['name'] _service.description = _service_desc['description'] _service.start = _service_desc['start'] _service.stop = _service_desc['stop'] _service.restart = _service_desc['restart'] _service.env = _service_desc.get('env', {}) _service.enable = _service_desc.get('enable', True) _service.autostart = _service_desc.get('autostart', True) store.add(_service) if 'directories' in bundle_desc: for _dir_desc in bundle_desc['directories']: _dir = store.find(Directory, Directory.name == _dir_desc['name'], Directory.bundle == bundle_desc['name']).one() or Directory() _dir.name = _dir_desc['name'] _dir.bundle = bundle_desc['name'] _dir.dir = _dir_desc.get('dir', None) _dir.permission = _dir_desc.get('permission', u'0755') _dir.description = _dir_desc['description'] store.add(_dir) store.commit() # copy data if 'data' in bundle_desc: _env = _bundle.env(True) data = bundle_desc['data'] src = data['src'].format(**_env) dst = data['dst'].format(**_env) if os.path.exists(dst) and len(os.listdir(dst)) == 0: logger.info('remove empty directory %s' % dst) shutil.rmtree(dst) if not os.path.exists(dst): shutil.copytree(src, dst) # # bin symlink bundle_bin_dir = runtime.path.join(env.get('dir_bundles'), _bundle.name, 'bin') if not os.path.exists(env.get('dir_bin')): os.makedirs(env.get('dir_bin')) if os.path.exists(bundle_bin_dir): for item in os.listdir(bundle_bin_dir): dst = runtime.path.join(env.get('dir_bin'), item) if item in ('.DS_Store'): continue if os.path.islink(dst) or os.path.isfile(dst): os.unlink(dst) os.symlink(runtime.path.join(bundle_bin_dir, item), dst) # # lib symlink bundle_lib_dir = runtime.path.join(env.get('dir_bundles'), _bundle.name, 'lib') lib_dir = runtime.path.join(env.get('dir_bundles'), 'lib') if not os.path.exists(lib_dir): os.makedirs(lib_dir) if os.path.exists(bundle_lib_dir): for item in os.listdir(bundle_lib_dir): src = runtime.path.join(bundle_lib_dir, item) dst = runtime.path.join(lib_dir, item) if item in ('.DS_Store'): continue if os.path.islink(dst) or os.path.isfile(dst): os.unlink(dst) if os.path.isdir(src): continue os.symlink(src, dst)