def _parse_configs(self):
mr_cluster = get_cluster_conf_for_job_submission()
if mr_cluster is None:
mapred_home = None
else:
mapred_home = mr_cluster.HADOOP_MAPRED_HOME.get()
shell_types = [
] # List of available shell types. For each shell type, we have a nice name (e.g. "Python Shell") and a short name (e.g. "python")
for item in shell.conf.SHELL_TYPES.keys():
if mapred_home:
env_for_shell = {constants.HADOOP_MAPRED_HOME: mapred_home}
else:
env_for_shell = {}
command = shell.conf.SHELL_TYPES[item].command.get().strip().split(
)
nice_name = shell.conf.SHELL_TYPES[item].nice_name.get().strip()
executable_exists = utils.executable_exists(command)
if executable_exists:
self._command_by_short_name[item] = command
conf_shell_env = shell.conf.SHELL_TYPES[item].environment
for env_variable in conf_shell_env.keys():
env_for_shell[env_variable] = conf_shell_env[
env_variable].value.get()
self._env_by_short_name[item] = env_for_shell
shell_types.append({
constants.NICE_NAME: nice_name,
constants.KEY_NAME: item,
constants.EXISTS: executable_exists
})
self.shell_types = shell_types
def test_rest():
reset_all_users()
username = pwd.getpwuid(os.getuid()).pw_name
client = make_logged_in_client(username=username, is_superuser=True)
d = {'eventlet.input': None}
response = client.get('/shell/', follow=True, **d)
shell_types_available = []
for item in shell.conf.SHELL_TYPES.keys():
nice_name = shell.conf.SHELL_TYPES[item].nice_name.get().strip()
assert nice_name in response.content
command = shell.conf.SHELL_TYPES[item].command.get().strip().split()
if utils.executable_exists(command):
shell_types_available.append(item)
if not shell_types_available:
return
response = client.get('/shell/create?keyName=%s' % (time.strftime("%s"), ),
follow=True,
**d)
assert "There is no shell with that name." in response.content
nonexistent_username = "******" % (time.strftime("%s"), )
client2 = make_logged_in_client(username=nonexistent_username,
is_superuser=True)
response = client2.get("/shell/create?keyName=%s" %
(shell_types_available[0], ),
follow=True,
**d)
assert "There is no Unix user account for you." in response.content
response = client.get("/shell/create?keyName=%s" %
(shell_types_available[0], ),
follow=True,
**d)
fragment = '<span id="shell_id" class="hide">'
assert fragment in response.content
shell_id_start = response.content.index(fragment) + len(fragment)
shell_id_end = response.content.index('</span>', shell_id_start)
shell_id = response.content[shell_id_start:shell_id_end]
assert re.match(r"^\s*\d+\s*$", shell_id)
response = client.post("/shell/kill_shell",
follow=True,
data={constants.SHELL_ID: shell_id},
**d)
assert 'Shell successfully marked for cleanup' in response.content
def config_validator():
"""
config_validator() -> [ (config_variable, error_message) ]
Called by core check_config() view.
"""
result = []
for item in SHELL_TYPES.keys():
command = SHELL_TYPES[item].command.get().strip().split()
nice_name = SHELL_TYPES[item].nice_name.get().strip()
if not utils.executable_exists(command):
result.append((SHELL_TYPES, "Command '%s' for entry '%s' in Shell app configuration cannot \
be found on the path." % (' '.join(command), item,) ,))
return result
def _parse_configs(self):
shell_types = [] # List of available shell types. For each shell type, we have a nice name (e.g. "Python Shell") and a short name (e.g. "python")
for item in shell.conf.SHELL_TYPES.keys():
env_for_shell = { constants.HADOOP_HOME: hadoop.conf.HADOOP_HOME.get() }
command = shell.conf.SHELL_TYPES[item].command.get().strip().split()
nice_name = shell.conf.SHELL_TYPES[item].nice_name.get().strip()
executable_exists = utils.executable_exists(command)
if executable_exists:
self._command_by_short_name[item] = command
conf_shell_env = shell.conf.SHELL_TYPES[item].environment
for env_variable in conf_shell_env.keys():
env_for_shell[env_variable] = conf_shell_env[env_variable].value.get()
self._env_by_short_name[item] = env_for_shell
shell_types.append({ constants.NICE_NAME: nice_name, constants.KEY_NAME: item, constants.EXISTS:executable_exists })
self.shell_types = shell_types
def config_validator():
"""
config_validator() -> [ (config_variable, error_message) ]
Called by core check_config() view.
"""
result = []
for item in SHELL_TYPES.keys():
command = SHELL_TYPES[item].command.get().strip().split()
nice_name = SHELL_TYPES[item].nice_name.get().strip()
if not utils.executable_exists(command):
result.append((
SHELL_TYPES,
"Command '%s' for entry '%s' in Shell app configuration cannot \
be found on the path." % (
' '.join(command),
item,
),
))
return result
def test_rest():
reset_all_users()
username = pwd.getpwuid(os.getuid()).pw_name
client = make_logged_in_client(username=username, is_superuser=True)
d = {"eventlet.input": None}
response = client.get("/shell/", follow=True, **d)
shell_types_available = []
for item in shell.conf.SHELL_TYPES.keys():
nice_name = shell.conf.SHELL_TYPES[item].nice_name.get().strip()
assert nice_name in response.content
command = shell.conf.SHELL_TYPES[item].command.get().strip().split()
if utils.executable_exists(command):
shell_types_available.append(item)
if not shell_types_available:
return
response = client.get("/shell/create?keyName=%s" % (time.strftime("%s"),), follow=True, **d)
assert "There is no shell with that name." in response.content
nonexistent_username = "******" % (time.strftime("%s"),)
client2 = make_logged_in_client(username=nonexistent_username, is_superuser=True)
response = client2.get("/shell/create?keyName=%s" % (shell_types_available[0],), follow=True, **d)
assert "There is no Unix user account for you." in response.content
response = client.get("/shell/create?keyName=%s" % (shell_types_available[0],), follow=True, **d)
fragment = '<span id="shell_id" class="hide">'
assert fragment in response.content
shell_id_start = response.content.index(fragment) + len(fragment)
shell_id_end = response.content.index("</span>", shell_id_start)
shell_id = response.content[shell_id_start:shell_id_end]
assert re.match(r"^\s*\d+\s*$", shell_id)
response = client.post("/shell/kill_shell", follow=True, data={constants.SHELL_ID: shell_id}, **d)
assert "Shell successfully marked for cleanup" in response.content
def __init__(self):
self._shells = {} # Keys are (username, shell_id) tuples. Each user has his/her own set of shell ids.
shell_types = [] # List of available shell types. For each shell type, we have a nice name (e.g. "Python Shell") and a short name (e.g. "python")
self._command_by_short_name = {} # Map each short name to its command (e.g. ["pig", "-l", "/dev/null"])
self._meta = {} # Map usernames to utils.UserMetadata objects
self._greenlets_by_hid = {} # Map each Hue Instance ID (HID) to greenlet currently fetching output for that HID.
self._hids_by_pid = {} # Map each process ID (PID) to the HID whose greenlet is currently doing a "select" on the process's output fd.
self._greenlets_to_notify = {} # For each PID, maintain a set of greenlets who are also interested in the output from that process, but are not doing the select.
self._shells_by_fds = {} # Map each file descriptor to the Shell instance whose output it represents.
self._greenlet_interruptable = {} # For each greenlet, store if it can be safely interrupted.
self._delegation_token_dir = shell.conf.SHELL_DELEGATION_TOKEN_DIR.get()
if not os.path.exists(self._delegation_token_dir):
os.mkdir(self._delegation_token_dir)
for item in shell.conf.SHELL_TYPES.keys():
command = shell.conf.SHELL_TYPES[item].command.get().strip().split()
nice_name = shell.conf.SHELL_TYPES[item].nice_name.get().strip()
executable_exists = utils.executable_exists(command)
if executable_exists:
self._command_by_short_name[item] = command
shell_types.append({ constants.NICE_NAME: nice_name, constants.KEY_NAME: item, constants.EXISTS:executable_exists })
self.shell_types = shell_types
eventlet.spawn_after(1, self._handle_periodic)
def _parse_configs(self):
mr_cluster = get_cluster_conf_for_job_submission()
if mr_cluster is None:
mapred_home = None
else:
mapred_home = mr_cluster.HADOOP_MAPRED_HOME.get()
shell_types = [] # List of available shell types. For each shell type, we have a nice name (e.g. "Python Shell") and a short name (e.g. "python")
for item in shell.conf.SHELL_TYPES.keys():
if mapred_home:
env_for_shell = { constants.HADOOP_MAPRED_HOME: mapred_home }
else:
env_for_shell = {}
command = shell.conf.SHELL_TYPES[item].command.get().strip().split()
nice_name = shell.conf.SHELL_TYPES[item].nice_name.get().strip()
executable_exists = utils.executable_exists(command)
if executable_exists:
self._command_by_short_name[item] = command
conf_shell_env = shell.conf.SHELL_TYPES[item].environment
for env_variable in conf_shell_env.keys():
env_for_shell[env_variable] = conf_shell_env[env_variable].value.get()
self._env_by_short_name[item] = env_for_shell
shell_types.append({ constants.NICE_NAME: nice_name, constants.KEY_NAME: item, constants.EXISTS:executable_exists })
self.shell_types = shell_types
