def execute_from_command_line(argv=None):
from security.command import CommandExecutor
def execute_from_command_line_with_stdout(argv=None,
stdout=None,
stderr=None):
try:
from django.core.management.base import BaseCommand
command_stdout = stdout
command_stderr = stderr
def command_init_patch(self,
stdout=None,
stderr=None,
no_color=False,
force_color=False):
stdout = command_stdout if stdout is None else stdout
stderr = command_stderr if stderr is None else stderr
self._patched_init(stdout=stdout,
stderr=stderr,
no_color=no_color,
force_color=force_color)
BaseCommand._patched_init = BaseCommand.__init__
BaseCommand.__init__ = command_init_patch
execute_from_command_line_original(argv=argv)
finally:
BaseCommand.__init__ = BaseCommand._patched_init
if len(argv) > 1:
# some arguments must be processed before django setup
parser_args = (None, ) if StrictVersion(
get_main_version()) < StrictVersion('2.1') else tuple()
parser = CommandParser(*parser_args,
usage='%(prog)s subcommand [options] [args]',
add_help=False)
parser.add_argument('--settings')
parser.add_argument('--pythonpath')
parser.add_argument('args', nargs='*') # catch-all
try:
options, args = parser.parse_known_args(argv[2:])
handle_default_options(options)
except CommandError:
pass # Ignore any option errors at this point.
django.setup()
return CommandExecutor(
command_function=execute_from_command_line_with_stdout,
command_kwargs={
'argv': argv
},
name=argv[1],
input=' '.join(argv[2:]),
is_executed_from_command_line=True).run()
else:
execute_from_command_line_original(argv=argv)
def get_local_test_settings_file(argv):
assert argv[1] == "test"
assert "manage.py" in argv[0]
local_settings_dir = os.path.join(
os.path.split(argv[0])[0], "local_settings")
from django.core.management import CommandParser, CommandError
parser = CommandParser(usage="%(prog)s subcommand [options] [args]",
add_help=False)
parser.add_argument('--local_test_settings', dest="local_test_settings")
options, args = parser.parse_known_args(argv)
if options.local_test_settings is None:
local_settings_file = "local_settings_example.py"
else:
local_settings_file = options.local_test_settings
if os.path.split(local_settings_file)[0] == "":
local_settings_file = os.path.join(local_settings_dir,
local_settings_file)
if os.path.abspath(local_settings_file) == os.path.abspath(
os.path.join(local_settings_dir, "local_settings.py")):
raise CommandError("Using production local_settings for tests is not "
"allowed due to security reason.")
if not os.path.isfile(local_settings_file):
raise CommandError("file '%s' does not exist" % local_settings_file)
return local_settings_file
def __call__(self, argv=None):
if argv is None:
argv = sys.argv[1:]
warnings.filterwarnings('ignore',
module="IPython",
category=DeprecationWarning)
warnings.filterwarnings("ignore", module="distutils")
try:
warnings.filterwarnings("ignore", category=ResourceWarning)
except NameError:
pass
warnings.filterwarnings("ignore", "invalid escape sequence",
DeprecationWarning)
# Ignore a deprecation warning in distutils.spawn for python 3.4+
if sys.version_info > (3, 4):
warnings.filterwarnings("ignore", "the imp module", Warning)
warnings.filterwarnings('once', 'Selenium support for PhantomJS',
Warning)
import django
from django.core.management import CommandParser, handle_default_options
# Ignore a python 3.6 DeprecationWarning in ModelBase.__new__ that isn't
# fixed in Django 1.x
if sys.version_info > (3, 6) and django.VERSION < (2, ):
warnings.filterwarnings("ignore", "__class__ not set defining",
DeprecationWarning)
parser = CommandParser(None,
usage="%(prog)s [options] [args]",
add_help=False)
default_settings = os.environ.get('DJANGO_SETTINGS_MODULE',
self.default_settings_module)
parser.add_argument('--settings', default=default_settings)
parser.add_argument('--pythonpath')
parser.add_argument('--testrunner',
action='store',
default='django_admin_testutils.DiscoverRunner')
parser.add_argument('args', nargs='*')
options, remaining_args = parser.parse_known_args(argv)
handle_default_options(options)
test_labels = options.args or [self.default_test_label]
flags = ['--testrunner=%s' % options.testrunner] + remaining_args
from django.conf import settings
if 'grappelli' in settings.INSTALLED_APPS:
# Grappelli uses the deprecated django.conf.urls.patterns, but we
# don't want this to fail our tests.
warnings.filterwarnings("ignore", "django.conf.urls.patterns",
Warning)
self.execute(flags, test_labels)
def execute_from_command_line(argv=None):
def execute_from_command_line_with_stdout(argv=None,
stdout=None,
stderr=None):
try:
if stdout:
sys.stdout = stdout
if stderr:
sys.stderr = stderr
execute_from_command_line_original(argv=argv)
finally:
if stdout:
sys.stdout = sys.__stdout__
if stderr:
sys.stderr = sys.__stderr__
if len(argv) > 1:
from security.utils import CommandLogger
# some arguments must be processed before django setup
parser_args = (None, ) if StrictVersion(
get_main_version()) < StrictVersion('2.1') else tuple()
parser = CommandParser(*parser_args,
usage='%(prog)s subcommand [options] [args]',
add_help=False)
parser.add_argument('--settings')
parser.add_argument('--pythonpath')
parser.add_argument('args', nargs='*') # catch-all
try:
options, args = parser.parse_known_args(argv[2:])
handle_default_options(options)
except CommandError:
pass # Ignore any option errors at this point.
django.setup()
return CommandLogger(
command_function=execute_from_command_line_with_stdout,
command_kwargs={
'argv': argv
},
name=argv[1],
input=' '.join(argv[2:]),
executed_from_command_line=True).run()
else:
execute_from_command_line_original(argv=argv)
def __call__(self, argv=None):
from django.conf import settings
from django.core.management import CommandParser, handle_default_options
if argv is None:
argv = sys.argv[1:]
try:
warnings.filterwarnings("ignore", category=ResourceWarning)
except NameError:
pass
warnings.filterwarnings('ignore',
module="IPython",
category=DeprecationWarning)
warnings.filterwarnings("ignore", module="distutils")
warnings.filterwarnings("ignore", "invalid escape sequence",
DeprecationWarning)
warnings.filterwarnings("ignore", "the imp module", Warning)
warnings.filterwarnings('once', 'Selenium support for PhantomJS',
Warning)
default_settings = os.environ.get('DJANGO_SETTINGS_MODULE',
self.default_settings_module)
parser = CommandParser(usage="%(prog)s [options] [args]",
add_help=False)
parser.add_argument('--settings', default=default_settings)
parser.add_argument('--pythonpath')
parser.add_argument('--testrunner',
action='store',
default='selenosis.DiscoverRunner')
parser.add_argument('args', nargs='*')
options, remaining_args = parser.parse_known_args(argv)
handle_default_options(options)
test_labels = options.args or [self.default_test_label]
flags = ['--testrunner=%s' % options.testrunner] + remaining_args
if 'grappelli' in settings.INSTALLED_APPS:
# Grappelli uses the deprecated django.conf.urls.patterns, but we
# don't want this to fail our tests.
warnings.filterwarnings("ignore", "django.conf.urls.patterns",
Warning)
self.execute(flags, test_labels)
def main():
"""Run administrative tasks."""
parser = CommandParser()
parser.add_argument("--env")
try:
env_option, rest = parser.parse_known_args(sys.argv)
sys.argv = rest
os.environ["env"] = env_option.env
except:
pass
os.environ.setdefault("DJANGO_SETTINGS_MODULE",
"secrets_manager_prototype.settings")
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?") from exc
execute_from_command_line(sys.argv)
def get_local_test_settings_file(argv):
assert argv[1] == "test"
assert "manage.py" in argv[0]
local_settings_dir = os.path.split(argv[0])[0]
assert os.path.isfile(os.path.join(local_settings_dir, "manage.py"))
from django.core.management import CommandParser, CommandError
parser = CommandParser(None, usage="%(prog)s subcommand [options] [args]",
add_help=False)
parser.add_argument('--local_test_settings',
dest="local_test_settings")
options, args = parser.parse_known_args(argv)
if options.local_test_settings is None:
local_settings_file = "local_settings_example.py"
else:
local_settings_file = options.local_test_settings
if os.path.split(local_settings_file)[0] == "":
local_settings_file = os.path.join(
local_settings_dir, local_settings_file)
if os.path.abspath(local_settings_file) == os.path.abspath(
os.path.join(local_settings_dir, "local_settings.py")):
raise CommandError(
"Using production local_settings for tests is not "
"allowed due to security reason."
)
if not os.path.isfile(local_settings_file):
raise CommandError(
"file '%s' does not exist" % local_settings_file
)
return local_settings_file
def entry():
_deprecation_check(sys.argv[0])
from django.core.exceptions import ImproperlyConfigured
from django.core.management import execute_from_command_line, find_commands
from django.core.management import CommandParser
from django.core.management.base import BaseCommand
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'desktop.settings')
cm_config_file = '/etc/cloudera-scm-agent/config.ini'
ld_path_orig = None
if "LD_LIBRARY_PATH" in os.environ.keys():
ld_path_orig = os.environ["LD_LIBRARY_PATH"]
# What's the subcommand being run?
# This code uses the same logic from django.core.management to handle command args
subcommand = None
if "--skip-reload" in sys.argv:
skip_reload = True
sys.argv.remove("--skip-reload")
else:
skip_reload = False
# Check if --cm-managed flag is set and strip it out
# to prevent from sending to subcommands
if "--cm-managed" in sys.argv:
sys.argv.remove("--cm-managed")
cm_managed = True
else:
cm_managed = False
if len(sys.argv) > 1:
subcommand = sys.argv[1]
parser = CommandParser(None,
usage="%(prog)s subcommand [options] [args]",
add_help=False)
parser.parse_known_args(sys.argv[2:])
if len(sys.argv) > 1:
prof_id = subcommand = sys.argv[1]
#Check if this is a CM managed cluster
if os.path.isfile(
cm_config_file) and not cm_managed and not skip_reload:
print "ALERT: This appears to be a CM Managed environment"
print "ALERT: HUE_CONF_DIR must be set when running hue commands in CM Managed environment"
print "ALERT: Please run 'hue <command> --cm-managed'"
else:
prof_id = str(os.getpid())
# CM managed configure env vars
if cm_managed:
import ConfigParser
from ConfigParser import NoOptionError
config = ConfigParser.RawConfigParser()
config.read(cm_config_file)
try:
cm_agent_run_dir = config.get(
'General', 'agent_wide_credential_cache_location')
except NoOptionError:
cm_agent_run_dir = '/var/run/cloudera-scm-agent'
pass
#Parse CM supervisor include file for Hue and set env vars
cm_supervisor_dir = cm_agent_run_dir + '/supervisor/include'
cm_process_dir = cm_agent_run_dir + '/process'
hue_env_conf = None
envline = None
cm_hue_string = "HUE_SERVER"
for file in os.listdir(cm_supervisor_dir):
if cm_hue_string in file:
hue_env_conf = file
hue_env_conf = cm_supervisor_dir + "/" + hue_env_conf
if hue_env_conf == None:
process_dirs = fnmatch.filter(os.listdir(cm_process_dir),
'*%s*' % cm_hue_string)
process_dirs.sort()
hue_process_dir = cm_process_dir + "/" + process_dirs[-1]
hue_env_conf = fnmatch.filter(os.listdir(hue_process_dir),
'supervisor.conf')[0]
hue_env_conf = hue_process_dir + "/" + hue_env_conf
if not hue_env_conf == None:
if os.path.isfile(hue_env_conf):
hue_env_conf_file = open(hue_env_conf, "r")
for line in hue_env_conf_file:
if "environment" in line:
envline = line
if "directory" in line:
empty, hue_conf_dir = line.split("directory=")
os.environ["HUE_CONF_DIR"] = hue_conf_dir.rstrip()
else:
print "This appears to be a CM managed cluster, but the"
print "supervisor/include file for Hue could not be found"
print "in order to successfully run commands that access"
print "the database you need to set the following env vars:"
print ""
print " export JAVA_HOME=<java_home>"
print " export HUE_CONF_DIR=\"%s/`ls -1 %s | grep %s | sort -n | tail -1 `\"" % (
cm_processs_dir, cm_process_dir, cm_hue_string)
print " export HUE_IGNORE_PASSWORD_SCRIPT_ERRORS=1"
print " export HUE_DATABASE_PASSWORD=<hueDBpassword>"
print "If using Oracle as your database:"
print " export LD_LIBRARY_PATH=/path/to/instantclient"
print ""
print "If the above does not work, make sure Hue has been started on this server."
if not envline == None:
empty, environment = envline.split("environment=")
for envvar in environment.split(","):
include_env_vars = ("HADOOP_C", "PARCEL", "SCM_DEFINES",
"LD_LIBRARY")
if any(include_env_var in envvar
for include_env_var in include_env_vars):
envkey, envval = envvar.split("=")
envval = envval.replace("'", "").rstrip()
os.environ[envkey] = envval
#Set JAVA_HOME
if "JAVA_HOME" not in os.environ.keys():
if os.path.isfile(
'/usr/lib64/cmf/service/common/cloudera-config.sh'):
locate_java = subprocess.Popen([
'bash', '-c',
'. /usr/lib64/cmf/service/common/cloudera-config.sh; locate_java_home'
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
elif os.path.isfile(
'/opt/cloudera/cm-agent/service/common/cloudera-config.sh'
):
locate_java = subprocess.Popen([
'bash', '-c',
'. /opt/cloudera/cm-agent/service/common/cloudera-config.sh; locate_java_home'
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
locate_java = None
JAVA_HOME = "UNKNOWN"
if locate_java is not None:
for line in iter(locate_java.stdout.readline, ''):
if 'JAVA_HOME' in line:
JAVA_HOME = line.rstrip().split('=')[1]
if JAVA_HOME != "UNKNOWN":
os.environ["JAVA_HOME"] = JAVA_HOME
if "JAVA_HOME" not in os.environ.keys():
print "JAVA_HOME must be set and can't be found, please set JAVA_HOME environment variable"
print " export JAVA_HOME=<java_home>"
sys.exit(1)
#Make sure we set Oracle Client if configured
if "LD_LIBRARY_PATH" not in os.environ.keys():
if "SCM_DEFINES_SCRIPTS" in os.environ.keys():
for scm_script in os.environ["SCM_DEFINES_SCRIPTS"].split(":"):
if "ORACLE" in scm_script:
if os.path.isfile(scm_script):
oracle_source = subprocess.Popen(
". %s; env" % scm_script,
stdout=subprocess.PIPE,
shell=True,
executable="/bin/bash")
for line in oracle_source.communicate(
)[0].splitlines():
if "LD_LIBRARY_PATH" in line:
var, oracle_ld_path = line.split("=")
os.environ[
"LD_LIBRARY_PATH"] = oracle_ld_path
if "LD_LIBRARY_PATH" not in os.environ.keys():
print "LD_LIBRARY_PATH can't be found, if you are using ORACLE for your Hue database"
print "then it must be set, if not, you can ignore"
print " export LD_LIBRARY_PATH=/path/to/instantclient"
if "LD_LIBRARY_PATH" in os.environ.keys():
if ld_path_orig is not None and ld_path_orig == os.environ[
"LD_LIBRARY_PATH"]:
skip_reload = True
if not skip_reload:
reload_with_cm_env(cm_managed)
try:
# Let django handle the normal execution
if os.getenv("DESKTOP_PROFILE"):
_profile(prof_id, lambda: execute_from_command_line(sys.argv))
else:
execute_from_command_line(sys.argv)
except ImproperlyConfigured, e:
if len(sys.argv) > 1 and sys.argv[
1] == 'is_db_alive' and 'oracle' in str(e).lower():
print >> sys.stderr, e # Oracle connector is improperly configured
sys.exit(10)
else:
raise e
def entry():
_deprecation_check(sys.argv[0])
from django.core.exceptions import ImproperlyConfigured
from django.core.management import execute_from_command_line, find_commands
from django.core.management import CommandParser
from django.core.management.base import BaseCommand
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'desktop.settings')
# What's the subcommand being run?
# This code uses the same logic from django.core.management to handle command args
subcommand = None
if len(sys.argv) > 1:
subcommand = sys.argv[1]
parser = CommandParser(None,
usage="%(prog)s subcommand [options] [args]",
add_help=False)
parser.parse_known_args(sys.argv[2:])
if len(sys.argv) > 1:
prof_id = subcommand = sys.argv[1]
commands_req_db = [
"changepassword", "createsuperuser", "clean_history_docs",
"convert_documents", "sync_documents", "dbshell", "dumpdata",
"loaddata", "shell", "migrate", "syncdb", "import_ldap_group",
"import_ldap_user", "sync_ldap_users_and_groups",
"useradmin_sync_with_unix"
]
if subcommand in commands_req_db:
#Check if this is a CM managed cluster
cm_config_file = '/etc/cloudera-scm-agent/config.ini'
if os.path.isfile(
cm_config_file) and "--cm-managed" not in sys.argv:
if not "HUE_CONF_DIR" in os.environ:
print "ALERT: This appears to be a CM Managed environment"
print "ALERT: HUE_CONF_DIR must be set when running hue commands in CM Managed environment"
print "ALERT: Please run 'hue <command> --cm-managed'"
else:
prof_id = str(os.getpid())
# Check if --cm-managed flag is set and strip it out
# to prevent from sending to subcommands
if "--cm-managed" in sys.argv:
sys.argv.remove("--cm-managed")
import ConfigParser
from ConfigParser import NoOptionError
config = ConfigParser.RawConfigParser()
config.read(cm_config_file)
try:
cm_supervisor_dir = config.get(
'General', 'agent_wide_credential_cache_location')
except NoOptionError:
cm_supervisor_dir = '/var/run/cloudera-scm-agent'
pass
#Parse CM supervisor include file for Hue and set env vars
cm_supervisor_dir = cm_supervisor_dir + '/supervisor/include'
hue_env_conf = None
envline = None
cm_hue_string = "HUE_SERVER"
for file in os.listdir(cm_supervisor_dir):
if cm_hue_string in file:
hue_env_conf = file
if not hue_env_conf == None:
if os.path.isfile(cm_supervisor_dir + "/" + hue_env_conf):
hue_env_conf_file = open(
cm_supervisor_dir + "/" + hue_env_conf, "r")
for line in hue_env_conf_file:
if "environment" in line:
envline = line
if "directory" in line:
empty, hue_conf_dir = line.split("directory=")
os.environ["HUE_CONF_DIR"] = hue_conf_dir.rstrip()
else:
print "This appears to be a CM managed cluster the"
print "supervisor/include file for Hue could not be found"
print "in order to successfully run commands that access"
print "the database you need to set the following env vars:"
print ""
print " export JAVA_HOME=<java_home>"
print " export HUE_CONF_DIR=\"/var/run/cloudera-scm-agent/process/\`ls -1 /var/run/cloudera-scm-agent/process | grep HUE_SERVER | sort -n | tail -1 \`\""
print " export HUE_IGNORE_PASSWORD_SCRIPT_ERRORS=1"
print " export HUE_DATABASE_PASSWORD=<hueDBpassword>"
if not envline == None:
empty, environment = envline.split("environment=")
for envvar in environment.split(","):
if "HADOOP_C" in envvar or "PARCEL" in envvar:
envkey, envval = envvar.split("=")
envval = envval.replace("'", "").rstrip()
os.environ[envkey] = envval
#Set JAVA_HOME:
if "JAVA_HOME" not in os.environ.keys():
parcel_dir = os.environ["PARCELS_ROOT"] + '/' + os.environ[
"PARCEL_DIRNAMES"]
bigtop_javahome = parcel_dir + '/lib/bigtop-utils/bigtop-detect-javahome'
if os.path.isfile(bigtop_javahome):
command = "/bin/bash -c \"source " + bigtop_javahome + " && env\""
proc = subprocess.Popen(command,
stdout=subprocess.PIPE,
shell=True)
for procline in proc.stdout:
(key, _, value) = procline.partition("=")
if key == "JAVA_HOME":
os.environ[key] = value.rstrip()
if "JAVA_HOME" not in os.environ.keys():
print "Not able to set JAVA_HOME. Please set manually:"
print " export JAVA_HOME=<java_home>"
try:
# Let django handle the normal execution
if os.getenv("DESKTOP_PROFILE"):
_profile(prof_id, lambda: execute_from_command_line(sys.argv))
else:
execute_from_command_line(sys.argv)
except ImproperlyConfigured, e:
if len(sys.argv) > 1 and sys.argv[
1] == 'is_db_alive' and 'oracle' in str(e).lower():
print >> sys.stderr, e # Oracle connector is improperly configured
sys.exit(10)
else:
raise e
def execute(self):
try:
subcommand = self.argv[1]
except IndexError:
subcommand = 'help' # Display help if no arguments were given.
# Preprocess options to extract --settings and --pythonpath.
# These options could affect the commands that are available, so they
# must be processed early.
# Compatible with the processing
django_version = django.get_version().split('.')
if int(django_version[0]) == 2 and int(django_version[1]) < 1:
parser = CommandParser(
None,
usage='%(prog)s subcommand [options] [args]',
add_help=False,
allow_abbrev=False)
else:
parser = CommandParser(
usage='%(prog)s subcommand [options] [args]',
add_help=False,
allow_abbrev=False)
parser.add_argument('--settings')
parser.add_argument('--pythonpath')
parser.add_argument('args', nargs='*') # catch-all
try:
options, args = parser.parse_known_args(self.argv[2:])
handle_default_options(options)
except CommandError:
pass # Ignore any option errors at this point.
try:
settings.INSTALLED_APPS
except ImproperlyConfigured as exc:
self.settings_exception = exc
except ImportError as exc:
self.settings_exception = exc
if settings.configured:
# Start the auto-reloading dev server even if the code is broken.
# The hardcoded condition is a code smell but we can't rely on a
# flag on the command class because we haven't located it yet.
if subcommand == 'runserver' and '--noreload' not in self.argv:
try:
autoreload.check_errors(django.setup)()
except Exception:
# The exception will be raised later in the child process
# started by the autoreloader. Pretend it didn't happen by
# loading an empty list of applications.
apps.all_models = defaultdict(OrderedDict)
apps.app_configs = OrderedDict()
apps.apps_ready = apps.models_ready = apps.ready = True
# Remove options not compatible with the built-in runserver
# (e.g. options for the contrib.staticfiles' runserver).
# Changes here require manually testing as described in
# #27522.
_parser = self.fetch_command('runserver').create_parser(
'django', 'runserver')
_options, _args = _parser.parse_known_args(self.argv[2:])
for _arg in _args:
self.argv.remove(_arg)
# In all other cases, django.setup() is required to succeed.
else:
django.setup()
self.autocomplete()
if subcommand == 'help':
if '--commands' in args:
sys.stdout.write(
self.main_help_text(commands_only=True) + '\n')
elif not options.args:
sys.stdout.write(self.main_help_text() + '\n')
else:
self.fetch_command(options.args[0]).print_help(
self.prog_name, options.args[0])
# Special-cases: We want 'django-admin --version' and
# 'django-admin --help' to work, for backwards compatibility.
elif subcommand == 'version' or self.argv[1:] == ['--version']:
sys.stdout.write(django.get_version() + '\n')
elif self.argv[1:] in (['--help'], ['-h']):
sys.stdout.write(self.main_help_text() + '\n')
else:
self.fetch_command(subcommand).run_from_argv(self.argv)
# module with --settings. If so, use the settings module for
# non-country-specific tests that we know has the right INSTALLED_APPS
# and tests are expected to pass with. This won't help confusion if
# someone uses "django-admin.py test" instead, but it's some help...
# (Note that if someone's set the DJANGO_SETTINGS_MODULE environment
# variable, this default won't be used either.)
try:
subcommand = sys.argv[1]
except IndexError:
subcommand = 'help'
parser = CommandParser(None)
parser.add_argument('--settings')
try:
options, args = parser.parse_known_args(sys.argv[2:])
except:
# Ignore any errors at this point; the arguments will be parsed
# again shortly anyway.
args = []
run_default_tests = (subcommand == 'test' and not options.settings)
if __name__ == "__main__":
if run_default_tests:
settings_module = 'mysite.settings.tests'
else:
settings_module = 'mysite.settings.base'
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings_module)
