def execute_from_command_line(argv=None): from security.command import CommandExecutor def execute_from_command_line_with_stdout(argv=None, stdout=None, stderr=None): try: from django.core.management.base import BaseCommand command_stdout = stdout command_stderr = stderr def command_init_patch(self, stdout=None, stderr=None, no_color=False, force_color=False): stdout = command_stdout if stdout is None else stdout stderr = command_stderr if stderr is None else stderr self._patched_init(stdout=stdout, stderr=stderr, no_color=no_color, force_color=force_color) BaseCommand._patched_init = BaseCommand.__init__ BaseCommand.__init__ = command_init_patch execute_from_command_line_original(argv=argv) finally: BaseCommand.__init__ = BaseCommand._patched_init if len(argv) > 1: # some arguments must be processed before django setup parser_args = (None, ) if StrictVersion( get_main_version()) < StrictVersion('2.1') else tuple() parser = CommandParser(*parser_args, usage='%(prog)s subcommand [options] [args]', add_help=False) parser.add_argument('--settings') parser.add_argument('--pythonpath') parser.add_argument('args', nargs='*') # catch-all try: options, args = parser.parse_known_args(argv[2:]) handle_default_options(options) except CommandError: pass # Ignore any option errors at this point. django.setup() return CommandExecutor( command_function=execute_from_command_line_with_stdout, command_kwargs={ 'argv': argv }, name=argv[1], input=' '.join(argv[2:]), is_executed_from_command_line=True).run() else: execute_from_command_line_original(argv=argv)
def get_local_test_settings_file(argv): assert argv[1] == "test" assert "manage.py" in argv[0] local_settings_dir = os.path.join( os.path.split(argv[0])[0], "local_settings") from django.core.management import CommandParser, CommandError parser = CommandParser(usage="%(prog)s subcommand [options] [args]", add_help=False) parser.add_argument('--local_test_settings', dest="local_test_settings") options, args = parser.parse_known_args(argv) if options.local_test_settings is None: local_settings_file = "local_settings_example.py" else: local_settings_file = options.local_test_settings if os.path.split(local_settings_file)[0] == "": local_settings_file = os.path.join(local_settings_dir, local_settings_file) if os.path.abspath(local_settings_file) == os.path.abspath( os.path.join(local_settings_dir, "local_settings.py")): raise CommandError("Using production local_settings for tests is not " "allowed due to security reason.") if not os.path.isfile(local_settings_file): raise CommandError("file '%s' does not exist" % local_settings_file) return local_settings_file
def __call__(self, argv=None): if argv is None: argv = sys.argv[1:] warnings.filterwarnings('ignore', module="IPython", category=DeprecationWarning) warnings.filterwarnings("ignore", module="distutils") try: warnings.filterwarnings("ignore", category=ResourceWarning) except NameError: pass warnings.filterwarnings("ignore", "invalid escape sequence", DeprecationWarning) # Ignore a deprecation warning in distutils.spawn for python 3.4+ if sys.version_info > (3, 4): warnings.filterwarnings("ignore", "the imp module", Warning) warnings.filterwarnings('once', 'Selenium support for PhantomJS', Warning) import django from django.core.management import CommandParser, handle_default_options # Ignore a python 3.6 DeprecationWarning in ModelBase.__new__ that isn't # fixed in Django 1.x if sys.version_info > (3, 6) and django.VERSION < (2, ): warnings.filterwarnings("ignore", "__class__ not set defining", DeprecationWarning) parser = CommandParser(None, usage="%(prog)s [options] [args]", add_help=False) default_settings = os.environ.get('DJANGO_SETTINGS_MODULE', self.default_settings_module) parser.add_argument('--settings', default=default_settings) parser.add_argument('--pythonpath') parser.add_argument('--testrunner', action='store', default='django_admin_testutils.DiscoverRunner') parser.add_argument('args', nargs='*') options, remaining_args = parser.parse_known_args(argv) handle_default_options(options) test_labels = options.args or [self.default_test_label] flags = ['--testrunner=%s' % options.testrunner] + remaining_args from django.conf import settings if 'grappelli' in settings.INSTALLED_APPS: # Grappelli uses the deprecated django.conf.urls.patterns, but we # don't want this to fail our tests. warnings.filterwarnings("ignore", "django.conf.urls.patterns", Warning) self.execute(flags, test_labels)
def execute_from_command_line(argv=None): def execute_from_command_line_with_stdout(argv=None, stdout=None, stderr=None): try: if stdout: sys.stdout = stdout if stderr: sys.stderr = stderr execute_from_command_line_original(argv=argv) finally: if stdout: sys.stdout = sys.__stdout__ if stderr: sys.stderr = sys.__stderr__ if len(argv) > 1: from security.utils import CommandLogger # some arguments must be processed before django setup parser_args = (None, ) if StrictVersion( get_main_version()) < StrictVersion('2.1') else tuple() parser = CommandParser(*parser_args, usage='%(prog)s subcommand [options] [args]', add_help=False) parser.add_argument('--settings') parser.add_argument('--pythonpath') parser.add_argument('args', nargs='*') # catch-all try: options, args = parser.parse_known_args(argv[2:]) handle_default_options(options) except CommandError: pass # Ignore any option errors at this point. django.setup() return CommandLogger( command_function=execute_from_command_line_with_stdout, command_kwargs={ 'argv': argv }, name=argv[1], input=' '.join(argv[2:]), executed_from_command_line=True).run() else: execute_from_command_line_original(argv=argv)
def __call__(self, argv=None): from django.conf import settings from django.core.management import CommandParser, handle_default_options if argv is None: argv = sys.argv[1:] try: warnings.filterwarnings("ignore", category=ResourceWarning) except NameError: pass warnings.filterwarnings('ignore', module="IPython", category=DeprecationWarning) warnings.filterwarnings("ignore", module="distutils") warnings.filterwarnings("ignore", "invalid escape sequence", DeprecationWarning) warnings.filterwarnings("ignore", "the imp module", Warning) warnings.filterwarnings('once', 'Selenium support for PhantomJS', Warning) default_settings = os.environ.get('DJANGO_SETTINGS_MODULE', self.default_settings_module) parser = CommandParser(usage="%(prog)s [options] [args]", add_help=False) parser.add_argument('--settings', default=default_settings) parser.add_argument('--pythonpath') parser.add_argument('--testrunner', action='store', default='selenosis.DiscoverRunner') parser.add_argument('args', nargs='*') options, remaining_args = parser.parse_known_args(argv) handle_default_options(options) test_labels = options.args or [self.default_test_label] flags = ['--testrunner=%s' % options.testrunner] + remaining_args if 'grappelli' in settings.INSTALLED_APPS: # Grappelli uses the deprecated django.conf.urls.patterns, but we # don't want this to fail our tests. warnings.filterwarnings("ignore", "django.conf.urls.patterns", Warning) self.execute(flags, test_labels)
def main(): """Run administrative tasks.""" parser = CommandParser() parser.add_argument("--env") try: env_option, rest = parser.parse_known_args(sys.argv) sys.argv = rest os.environ["env"] = env_option.env except: pass os.environ.setdefault("DJANGO_SETTINGS_MODULE", "secrets_manager_prototype.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: raise ImportError( "Couldn't import Django. Are you sure it's installed and " "available on your PYTHONPATH environment variable? Did you " "forget to activate a virtual environment?") from exc execute_from_command_line(sys.argv)
def get_local_test_settings_file(argv): assert argv[1] == "test" assert "manage.py" in argv[0] local_settings_dir = os.path.split(argv[0])[0] assert os.path.isfile(os.path.join(local_settings_dir, "manage.py")) from django.core.management import CommandParser, CommandError parser = CommandParser(None, usage="%(prog)s subcommand [options] [args]", add_help=False) parser.add_argument('--local_test_settings', dest="local_test_settings") options, args = parser.parse_known_args(argv) if options.local_test_settings is None: local_settings_file = "local_settings_example.py" else: local_settings_file = options.local_test_settings if os.path.split(local_settings_file)[0] == "": local_settings_file = os.path.join( local_settings_dir, local_settings_file) if os.path.abspath(local_settings_file) == os.path.abspath( os.path.join(local_settings_dir, "local_settings.py")): raise CommandError( "Using production local_settings for tests is not " "allowed due to security reason." ) if not os.path.isfile(local_settings_file): raise CommandError( "file '%s' does not exist" % local_settings_file ) return local_settings_file
def entry(): _deprecation_check(sys.argv[0]) from django.core.exceptions import ImproperlyConfigured from django.core.management import execute_from_command_line, find_commands from django.core.management import CommandParser from django.core.management.base import BaseCommand os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'desktop.settings') cm_config_file = '/etc/cloudera-scm-agent/config.ini' ld_path_orig = None if "LD_LIBRARY_PATH" in os.environ.keys(): ld_path_orig = os.environ["LD_LIBRARY_PATH"] # What's the subcommand being run? # This code uses the same logic from django.core.management to handle command args subcommand = None if "--skip-reload" in sys.argv: skip_reload = True sys.argv.remove("--skip-reload") else: skip_reload = False # Check if --cm-managed flag is set and strip it out # to prevent from sending to subcommands if "--cm-managed" in sys.argv: sys.argv.remove("--cm-managed") cm_managed = True else: cm_managed = False if len(sys.argv) > 1: subcommand = sys.argv[1] parser = CommandParser(None, usage="%(prog)s subcommand [options] [args]", add_help=False) parser.parse_known_args(sys.argv[2:]) if len(sys.argv) > 1: prof_id = subcommand = sys.argv[1] #Check if this is a CM managed cluster if os.path.isfile( cm_config_file) and not cm_managed and not skip_reload: print "ALERT: This appears to be a CM Managed environment" print "ALERT: HUE_CONF_DIR must be set when running hue commands in CM Managed environment" print "ALERT: Please run 'hue <command> --cm-managed'" else: prof_id = str(os.getpid()) # CM managed configure env vars if cm_managed: import ConfigParser from ConfigParser import NoOptionError config = ConfigParser.RawConfigParser() config.read(cm_config_file) try: cm_agent_run_dir = config.get( 'General', 'agent_wide_credential_cache_location') except NoOptionError: cm_agent_run_dir = '/var/run/cloudera-scm-agent' pass #Parse CM supervisor include file for Hue and set env vars cm_supervisor_dir = cm_agent_run_dir + '/supervisor/include' cm_process_dir = cm_agent_run_dir + '/process' hue_env_conf = None envline = None cm_hue_string = "HUE_SERVER" for file in os.listdir(cm_supervisor_dir): if cm_hue_string in file: hue_env_conf = file hue_env_conf = cm_supervisor_dir + "/" + hue_env_conf if hue_env_conf == None: process_dirs = fnmatch.filter(os.listdir(cm_process_dir), '*%s*' % cm_hue_string) process_dirs.sort() hue_process_dir = cm_process_dir + "/" + process_dirs[-1] hue_env_conf = fnmatch.filter(os.listdir(hue_process_dir), 'supervisor.conf')[0] hue_env_conf = hue_process_dir + "/" + hue_env_conf if not hue_env_conf == None: if os.path.isfile(hue_env_conf): hue_env_conf_file = open(hue_env_conf, "r") for line in hue_env_conf_file: if "environment" in line: envline = line if "directory" in line: empty, hue_conf_dir = line.split("directory=") os.environ["HUE_CONF_DIR"] = hue_conf_dir.rstrip() else: print "This appears to be a CM managed cluster, but the" print "supervisor/include file for Hue could not be found" print "in order to successfully run commands that access" print "the database you need to set the following env vars:" print "" print " export JAVA_HOME=<java_home>" print " export HUE_CONF_DIR=\"%s/`ls -1 %s | grep %s | sort -n | tail -1 `\"" % ( cm_processs_dir, cm_process_dir, cm_hue_string) print " export HUE_IGNORE_PASSWORD_SCRIPT_ERRORS=1" print " export HUE_DATABASE_PASSWORD=<hueDBpassword>" print "If using Oracle as your database:" print " export LD_LIBRARY_PATH=/path/to/instantclient" print "" print "If the above does not work, make sure Hue has been started on this server." if not envline == None: empty, environment = envline.split("environment=") for envvar in environment.split(","): include_env_vars = ("HADOOP_C", "PARCEL", "SCM_DEFINES", "LD_LIBRARY") if any(include_env_var in envvar for include_env_var in include_env_vars): envkey, envval = envvar.split("=") envval = envval.replace("'", "").rstrip() os.environ[envkey] = envval #Set JAVA_HOME if "JAVA_HOME" not in os.environ.keys(): if os.path.isfile( '/usr/lib64/cmf/service/common/cloudera-config.sh'): locate_java = subprocess.Popen([ 'bash', '-c', '. /usr/lib64/cmf/service/common/cloudera-config.sh; locate_java_home' ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) elif os.path.isfile( '/opt/cloudera/cm-agent/service/common/cloudera-config.sh' ): locate_java = subprocess.Popen([ 'bash', '-c', '. /opt/cloudera/cm-agent/service/common/cloudera-config.sh; locate_java_home' ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: locate_java = None JAVA_HOME = "UNKNOWN" if locate_java is not None: for line in iter(locate_java.stdout.readline, ''): if 'JAVA_HOME' in line: JAVA_HOME = line.rstrip().split('=')[1] if JAVA_HOME != "UNKNOWN": os.environ["JAVA_HOME"] = JAVA_HOME if "JAVA_HOME" not in os.environ.keys(): print "JAVA_HOME must be set and can't be found, please set JAVA_HOME environment variable" print " export JAVA_HOME=<java_home>" sys.exit(1) #Make sure we set Oracle Client if configured if "LD_LIBRARY_PATH" not in os.environ.keys(): if "SCM_DEFINES_SCRIPTS" in os.environ.keys(): for scm_script in os.environ["SCM_DEFINES_SCRIPTS"].split(":"): if "ORACLE" in scm_script: if os.path.isfile(scm_script): oracle_source = subprocess.Popen( ". %s; env" % scm_script, stdout=subprocess.PIPE, shell=True, executable="/bin/bash") for line in oracle_source.communicate( )[0].splitlines(): if "LD_LIBRARY_PATH" in line: var, oracle_ld_path = line.split("=") os.environ[ "LD_LIBRARY_PATH"] = oracle_ld_path if "LD_LIBRARY_PATH" not in os.environ.keys(): print "LD_LIBRARY_PATH can't be found, if you are using ORACLE for your Hue database" print "then it must be set, if not, you can ignore" print " export LD_LIBRARY_PATH=/path/to/instantclient" if "LD_LIBRARY_PATH" in os.environ.keys(): if ld_path_orig is not None and ld_path_orig == os.environ[ "LD_LIBRARY_PATH"]: skip_reload = True if not skip_reload: reload_with_cm_env(cm_managed) try: # Let django handle the normal execution if os.getenv("DESKTOP_PROFILE"): _profile(prof_id, lambda: execute_from_command_line(sys.argv)) else: execute_from_command_line(sys.argv) except ImproperlyConfigured, e: if len(sys.argv) > 1 and sys.argv[ 1] == 'is_db_alive' and 'oracle' in str(e).lower(): print >> sys.stderr, e # Oracle connector is improperly configured sys.exit(10) else: raise e
def entry(): _deprecation_check(sys.argv[0]) from django.core.exceptions import ImproperlyConfigured from django.core.management import execute_from_command_line, find_commands from django.core.management import CommandParser from django.core.management.base import BaseCommand os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'desktop.settings') # What's the subcommand being run? # This code uses the same logic from django.core.management to handle command args subcommand = None if len(sys.argv) > 1: subcommand = sys.argv[1] parser = CommandParser(None, usage="%(prog)s subcommand [options] [args]", add_help=False) parser.parse_known_args(sys.argv[2:]) if len(sys.argv) > 1: prof_id = subcommand = sys.argv[1] commands_req_db = [ "changepassword", "createsuperuser", "clean_history_docs", "convert_documents", "sync_documents", "dbshell", "dumpdata", "loaddata", "shell", "migrate", "syncdb", "import_ldap_group", "import_ldap_user", "sync_ldap_users_and_groups", "useradmin_sync_with_unix" ] if subcommand in commands_req_db: #Check if this is a CM managed cluster cm_config_file = '/etc/cloudera-scm-agent/config.ini' if os.path.isfile( cm_config_file) and "--cm-managed" not in sys.argv: if not "HUE_CONF_DIR" in os.environ: print "ALERT: This appears to be a CM Managed environment" print "ALERT: HUE_CONF_DIR must be set when running hue commands in CM Managed environment" print "ALERT: Please run 'hue <command> --cm-managed'" else: prof_id = str(os.getpid()) # Check if --cm-managed flag is set and strip it out # to prevent from sending to subcommands if "--cm-managed" in sys.argv: sys.argv.remove("--cm-managed") import ConfigParser from ConfigParser import NoOptionError config = ConfigParser.RawConfigParser() config.read(cm_config_file) try: cm_supervisor_dir = config.get( 'General', 'agent_wide_credential_cache_location') except NoOptionError: cm_supervisor_dir = '/var/run/cloudera-scm-agent' pass #Parse CM supervisor include file for Hue and set env vars cm_supervisor_dir = cm_supervisor_dir + '/supervisor/include' hue_env_conf = None envline = None cm_hue_string = "HUE_SERVER" for file in os.listdir(cm_supervisor_dir): if cm_hue_string in file: hue_env_conf = file if not hue_env_conf == None: if os.path.isfile(cm_supervisor_dir + "/" + hue_env_conf): hue_env_conf_file = open( cm_supervisor_dir + "/" + hue_env_conf, "r") for line in hue_env_conf_file: if "environment" in line: envline = line if "directory" in line: empty, hue_conf_dir = line.split("directory=") os.environ["HUE_CONF_DIR"] = hue_conf_dir.rstrip() else: print "This appears to be a CM managed cluster the" print "supervisor/include file for Hue could not be found" print "in order to successfully run commands that access" print "the database you need to set the following env vars:" print "" print " export JAVA_HOME=<java_home>" print " export HUE_CONF_DIR=\"/var/run/cloudera-scm-agent/process/\`ls -1 /var/run/cloudera-scm-agent/process | grep HUE_SERVER | sort -n | tail -1 \`\"" print " export HUE_IGNORE_PASSWORD_SCRIPT_ERRORS=1" print " export HUE_DATABASE_PASSWORD=<hueDBpassword>" if not envline == None: empty, environment = envline.split("environment=") for envvar in environment.split(","): if "HADOOP_C" in envvar or "PARCEL" in envvar: envkey, envval = envvar.split("=") envval = envval.replace("'", "").rstrip() os.environ[envkey] = envval #Set JAVA_HOME: if "JAVA_HOME" not in os.environ.keys(): parcel_dir = os.environ["PARCELS_ROOT"] + '/' + os.environ[ "PARCEL_DIRNAMES"] bigtop_javahome = parcel_dir + '/lib/bigtop-utils/bigtop-detect-javahome' if os.path.isfile(bigtop_javahome): command = "/bin/bash -c \"source " + bigtop_javahome + " && env\"" proc = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) for procline in proc.stdout: (key, _, value) = procline.partition("=") if key == "JAVA_HOME": os.environ[key] = value.rstrip() if "JAVA_HOME" not in os.environ.keys(): print "Not able to set JAVA_HOME. Please set manually:" print " export JAVA_HOME=<java_home>" try: # Let django handle the normal execution if os.getenv("DESKTOP_PROFILE"): _profile(prof_id, lambda: execute_from_command_line(sys.argv)) else: execute_from_command_line(sys.argv) except ImproperlyConfigured, e: if len(sys.argv) > 1 and sys.argv[ 1] == 'is_db_alive' and 'oracle' in str(e).lower(): print >> sys.stderr, e # Oracle connector is improperly configured sys.exit(10) else: raise e
def execute(self): try: subcommand = self.argv[1] except IndexError: subcommand = 'help' # Display help if no arguments were given. # Preprocess options to extract --settings and --pythonpath. # These options could affect the commands that are available, so they # must be processed early. # Compatible with the processing django_version = django.get_version().split('.') if int(django_version[0]) == 2 and int(django_version[1]) < 1: parser = CommandParser( None, usage='%(prog)s subcommand [options] [args]', add_help=False, allow_abbrev=False) else: parser = CommandParser( usage='%(prog)s subcommand [options] [args]', add_help=False, allow_abbrev=False) parser.add_argument('--settings') parser.add_argument('--pythonpath') parser.add_argument('args', nargs='*') # catch-all try: options, args = parser.parse_known_args(self.argv[2:]) handle_default_options(options) except CommandError: pass # Ignore any option errors at this point. try: settings.INSTALLED_APPS except ImproperlyConfigured as exc: self.settings_exception = exc except ImportError as exc: self.settings_exception = exc if settings.configured: # Start the auto-reloading dev server even if the code is broken. # The hardcoded condition is a code smell but we can't rely on a # flag on the command class because we haven't located it yet. if subcommand == 'runserver' and '--noreload' not in self.argv: try: autoreload.check_errors(django.setup)() except Exception: # The exception will be raised later in the child process # started by the autoreloader. Pretend it didn't happen by # loading an empty list of applications. apps.all_models = defaultdict(OrderedDict) apps.app_configs = OrderedDict() apps.apps_ready = apps.models_ready = apps.ready = True # Remove options not compatible with the built-in runserver # (e.g. options for the contrib.staticfiles' runserver). # Changes here require manually testing as described in # #27522. _parser = self.fetch_command('runserver').create_parser( 'django', 'runserver') _options, _args = _parser.parse_known_args(self.argv[2:]) for _arg in _args: self.argv.remove(_arg) # In all other cases, django.setup() is required to succeed. else: django.setup() self.autocomplete() if subcommand == 'help': if '--commands' in args: sys.stdout.write( self.main_help_text(commands_only=True) + '\n') elif not options.args: sys.stdout.write(self.main_help_text() + '\n') else: self.fetch_command(options.args[0]).print_help( self.prog_name, options.args[0]) # Special-cases: We want 'django-admin --version' and # 'django-admin --help' to work, for backwards compatibility. elif subcommand == 'version' or self.argv[1:] == ['--version']: sys.stdout.write(django.get_version() + '\n') elif self.argv[1:] in (['--help'], ['-h']): sys.stdout.write(self.main_help_text() + '\n') else: self.fetch_command(subcommand).run_from_argv(self.argv)
# module with --settings. If so, use the settings module for # non-country-specific tests that we know has the right INSTALLED_APPS # and tests are expected to pass with. This won't help confusion if # someone uses "django-admin.py test" instead, but it's some help... # (Note that if someone's set the DJANGO_SETTINGS_MODULE environment # variable, this default won't be used either.) try: subcommand = sys.argv[1] except IndexError: subcommand = 'help' parser = CommandParser(None) parser.add_argument('--settings') try: options, args = parser.parse_known_args(sys.argv[2:]) except: # Ignore any errors at this point; the arguments will be parsed # again shortly anyway. args = [] run_default_tests = (subcommand == 'test' and not options.settings) if __name__ == "__main__": if run_default_tests: settings_module = 'mysite.settings.tests' else: settings_module = 'mysite.settings.base' os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings_module)