def run_installed_tests_as_subprocess(app_dir_path, django_settings_module,
python_exe_path=sys.executable,
use_logger=None, read_config_file=True):
"""Run the installed mode tests as a separate subprocess, using the installed app's environment.
A results file is used to communcate back the results of the validation. After the run is complete,
an instance of ParsedJsonResults is returned
"""
logger.info(">> Validating application settings")
app_dir_path = os.path.abspath(os.path.expanduser(app_dir_path))
if not os.path.exists(app_dir_path):
raise TestSetupError("Django settings file validator: application directory '%s' does not exist" % app_dir_path)
python_path_dir = find_python_module(django_settings_module, app_dir_path)
if python_path_dir==None:
raise TestSetupError("Django settings file validator: unable to find settings module %s under %s" %
(django_settings_module, app_dir_path))
if use_logger == None:
use_logger = logger
python_path = get_python_path(python_path_dir, django_settings_module)
use_logger.debug("Setting PYTHONPATH to %s" % python_path)
# we create a temporary file that will contain the results
tf = tempfile.NamedTemporaryFile(delete=False)
tf.write("null")
tf.close()
script_file = os.path.dirname(os.path.abspath(__file__))
try:
program_and_args = [python_exe_path, script_file, "-v",
"validate-installed", app_dir_path, tf.name]
if not read_config_file:
program_and_args.append(django_settings_module)
use_logger.debug(' '.join(program_and_args))
env = {
"PYTHONPATH": python_path,
"DJANGO_SETTINGS_MODULE": "%s" % \
get_deployed_settings_module(django_settings_module)
}
cwd = get_settings_file_directory(python_path_dir, django_settings_module)
subproc = subprocess.Popen(program_and_args,
env=env, stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, cwd=cwd)
use_logger.debug("Started program %s, pid is %d" % (program_and_args[0],
subproc.pid))
subproc.stdin.close()
for line in subproc.stdout:
use_logger.debug("[%d] %s" % (subproc.pid, line.rstrip()))
subproc.wait()
use_logger.debug("[%d] %s exited with return code %d" % (subproc.pid,
program_and_args[0],
subproc.returncode))
with open(tf.name, "rb") as f:
json_obj = json.load(f)
results = ParsedJsonResults(json_obj)
return results
finally:
os.remove(tf.name)
def validate_settings(app_dir_path, django_settings_module, django_config=None,
prev_version_component_list=None):
"""This is the main settings validation function. It takes the following arguments:
app_dir_path - path to the top level directory of the extracted application
django_settings_module - fully qualified name of the django settings module
django_config - if provided, this is the django_config data generated
during the original packaging of the app. We validate
that it is still consistent with the current app.
This function returns an intance of SettingValidationResults.
Note that validate_settings() must be run after generate_settings(). We import
the deployed_settings module rather than the user's django_settings_module so
we can see whether they've overridden any of the settings.
"""
# normalize the target install path
app_dir_path = os.path.abspath(os.path.expanduser(app_dir_path))
results = SettingValidationResults(VERSION, logger)
python_path_dir = find_python_module(django_settings_module, app_dir_path)
if not python_path_dir:
raise ValidationError("Unable to find django settings module %s under %s" % (django_settings_module, app_dir_path))
# we store only the subdirectory part of the python path, since the rest depends
# on where we install the app.
if os.path.dirname(app_dir_path)==python_path_dir:
results.python_path_subdirectory = ""
else:
results.python_path_subdirectory = _get_subdir_component(os.path.dirname(app_dir_path), python_path_dir)
# get the settings file directory
settings_file_directory = get_settings_file_directory(python_path_dir, django_settings_module)
# do the import of app's settings
sys.path = [python_path_dir] + sys.path
deployed_settings_module = get_deployed_settings_module(django_settings_module)
logger.debug("Importing settings module %s" % deployed_settings_module)
try:
settings_module = import_module(deployed_settings_module)
except:
(exc_type, exc_value, exc_traceback) = sys.exc_info()
logger.exception("Exception in settings file import: %s(%s)" %
(exc_type.__name__, str(exc_value)))
raise SettingsImportError("Error in settings import: %s(%s)" %
(exc_type.__name__, str(exc_value)))
# Check that the settings controlled by engage weren't overridden by app.
# If any are overridden, we treat them as warnings.
check_if_setting_overridden('TIME_ZONE', settings_module, results)
check_if_setting_overridden('SECRET_KEY', settings_module, results)
check_if_setting_overridden('ADMINS', settings_module, results)
check_if_setting_overridden('DATABASES', settings_module, results)
check_if_setting_overridden('LOGGING_CONFIG', settings_module,
results)
# Check that settings which point to a directory are either not set or
# point to a valid directory
if hasattr(settings_module, "MEDIA_ROOT"):
check_directory_setting("MEDIA_ROOT",
settings_module.MEDIA_ROOT,
'', app_dir_path, results)
if hasattr(settings_module, "TEMPLATE_DIRS"):
check_directory_tuple_setting("TEMPLATE_DIRS",
settings_module.TEMPLATE_DIRS,
app_dir_path, results)
# Get the packages in requirements.txt. We use this in validating
# the django apps. We defer the validation of the actual packages
# until we have parsed and validated the engage_components.json file.
user_required_packages = get_user_required_packages(app_dir_path)
# check that all INSTALLED_APPS are pointing to apps accessible in the target system
if hasattr(settings_module, "INSTALLED_APPS"):
installed_apps = []
packages = PREINSTALLED_PACKAGES + user_required_packages
known_apps = set(get_apps_for_packages(packages))
for app_name in settings_module.INSTALLED_APPS:
validate_installed_app(app_name, python_path_dir, known_apps,
app_dir_path, django_settings_module, results)
installed_apps.append(app_name)
else:
installed_apps = []
results.installed_apps = installed_apps
if hasattr(settings_module, "FIXTURE_DIRS"):
fixture_dirs = _tuple_setting_to_list(settings_module.FIXTURE_DIRS)
check_directory_tuple_setting("FIXTURE_DIRS", fixture_dirs,
app_dir_path, results)
else:
fixture_dirs = []
# check that ENGAGE_APP_DB_FIXTURES points to valid fixture files
if hasattr(settings_module, "ENGAGE_APP_DB_FIXTURES"):
results.fixtures = _tuple_setting_to_list(settings_module.ENGAGE_APP_DB_FIXTURES)
for fixture in results.fixtures:
validate_fixture_file(fixture, results.installed_apps, fixture_dirs,
python_path_dir, settings_file_directory, known_apps, results)
else:
results.fixtures = []
# check ENGAGE_MIGRATION_APPS, if present
if hasattr(settings_module, "ENGAGE_MIGRATION_APPS"):
results.migration_apps = _tuple_setting_to_list(settings_module.ENGAGE_MIGRATION_APPS)
if len(results.migration_apps)>0 and not ("south" in results.installed_apps):
results.error("Django apps to upgraded specified in ENGAGE_MIGRATION_APPS, but south not included in INSTALLED_APPS")
validate_migration_apps(results.migration_apps, results.installed_apps, results)
else:
results.migration_apps = []
# check the static files directories, if present. Each entry could be a source
# directory, or a tuple of (target_subdir, source_path)
if hasattr(settings_module, "STATICFILES_DIRS"):
staticfiles_dirs = _tuple_setting_to_list(settings_module.STATICFILES_DIRS)
for dirpath in staticfiles_dirs:
if isinstance(dirpath, tuple):
dirpath = dirpath[1]
if not os.path.isdir(dirpath):
results.error("Setting STATICFILES_DIRS references '%s', which does not exist" % dirpath)
elif string.find(os.path.realpath(dirpath),
os.path.realpath(app_dir_path)) != 0:
results.error("Setting STATICFILES_DIRS references '%s', which is not a subdirectory of '%s'" % (dirpath, app_dir_path))
check_directory_tuple_setting("STATICFILES_DIRS", staticfiles_dirs,
app_dir_path, results)
# gather the values of static files related settings for use during
# installation.
extract_static_files_settings(settings_module, app_dir_path, results)
# check each command in ENGAGE_DJANGO_POSTINSTALL_COMMANDS is actually present in manager
if hasattr(settings_module, "ENGAGE_DJANGO_POSTINSTALL_COMMANDS"):
results.post_install_commands = list(settings_module.ENGAGE_DJANGO_POSTINSTALL_COMMANDS)
validate_post_install_commands(app_name, settings_module, results)
else:
results.post_install_commands = []
# read the additional components file and put the data into the results
additional_comp_file = os.path.join(app_dir_path, COMPONENTS_FILENAME)
if os.path.exists(additional_comp_file):
with open(additional_comp_file, "rb") as cf:
results.components = read_components_file(cf, additional_comp_file, None)
else:
results.components = []
# validate the user required packages, taking into account the components requested
# by the user.
validate_package_list(user_required_packages, results.components, results)
# extract the product name and version, if present
if hasattr(settings_module, "ENGAGE_PRODUCT_NAME"):
results.product = settings_module.ENGAGE_PRODUCT_NAME
if hasattr(settings_module, "ENGAGE_PRODUCT_VERSION"):
results.product_version = settings_module.ENGAGE_PRODUCT_VERSION
# if provided, check that the django_config matches the settings values
if django_config:
django_config_ok = True
if installed_apps != django_config.installed_apps:
results.error("INSTALLED_APPS in configuration file (%s) does not match INSTALLED_APPS in settings file (%s). Your configuration file is likely out of date. Try re-running prepare." %
(django_config.installed_apps.__repr__(),
installed_apps.__repr__()))
django_config_ok = False
if results.fixtures != django_config.fixtures:
# TODO: this was originally an error, which caused some issues.
# See ticket #166.
results.warning("ENGAGE_APP_DB_FIXTURES in configuration file (%s) does not match value in settings file (%s). If this is not what you expect, your configuration file is likely out of date: try re-running prepare." %
(django_config.fixtures.__repr__(),
results.fixtures.__repr__()))
django_config_ok = False
if results.migration_apps != django_config.migration_apps:
results.error("ENGAGE_MIGRATION_APPS in configuration file (%s) does not match value in settings file (%s). Your configuration file is likely out of date. Try re-running prepare." %
(django_config.migration_apps.__repr__(),
results.migration_apps.__repr__()))
django_config_ok = False
if results.product and results.product != django_config.product:
results.error("ENGAGE_PRODUCT_NAME in configuration file (%s) does not match value in settings file (%s). Your configuration file is likely out of date. Try re-running prepare." % (django_config.product, results.product))
django_config_ok = False
if results.product_version and results.product_version != django_config.product_version:
results.error("ENGAGE_PRODUCT_VERSION in configuration file (%s) does not match value in settings file (%s). Your configuration file is likely out of date. Try re-running prepare." % (django_config.product_version, results.product_version))
django_config_ok = False
if django_config_ok:
logger.debug("Verified config file is consistent with settings file")
return results # all done
def generate_settings_file(app_dir_path, django_settings_module, components_list,
properties=None):
"""Generate the engage_settings.py and deployed_settings.py files. Returns a
list of operations which will put the files back to their original state,
where each operation is a (function, arglist) tuple.
"""
undo_ops = []
# first, get the location of the python path.
python_path = find_python_module(django_settings_module, app_dir_path)
if not python_path:
raise FileMissingError("Cannot find django settings module %s under %s" %
(django_settings_module, os.path.basename(app_dir_path)))
# compute the paths for the engage_settings.py and deployed_settings.py modules
subdirs = django_settings_module.split('.')
if len(subdirs)==1:
parent_dir = python_path
engage_settings_module = "engage_settings"
deployed_settings_module = "deployed_settings"
else:
parent_dir = os.path.join(python_path, "/".join(subdirs[0:-1]))
parent_module = ".".join(subdirs[0:-1])
engage_settings_module = parent_module + ".engage_settings"
deployed_settings_module = parent_module + "deployed_settings"
assert os.path.exists(parent_dir), "Internal error: could not find directory %s" % parent_dir
engage_settings_py = os.path.join(parent_dir, "engage_settings.py")
deployed_settings_py = os.path.join(parent_dir, "deployed_settings.py")
# warn if the generated files already exist
if os.path.exists(engage_settings_py):
logger.warn("%s already exists, will be overwritten" % engage_settings_py)
if os.path.exists(deployed_settings_py):
logger.warn("%s already exists, will be overwritten" % deployed_settings_py)
lowered_components_list = [unicode.lower(s) for s in components_list]
# now generate the engage settings file
if not properties:
properties = {
INSTALL_PATH: os.path.dirname(app_dir_path),
ADMIN_NAME: "admin_" + _gen_rand_string(5),
ADMIN_EMAIL: "admn" + _gen_rand_string(5) + "@foo.com",
SECRET_KEY: _gen_rand_string(50, chars=string.digits+string.letters+"!#%&()*+,-./:;<=>?@[]^_`{|}~"),
DATABASE_ENGINE: 'django.db.backends.sqlite3',
DATABASE_NAME: os.path.join(os.path.join(parent_dir, "db"), "django.db"),
DATABASE_USER: '',
DATABASE_PASSWORD: '',
DATABASE_HOST: '',
DATABASE_PORT: None,
CACHE_BACKEND: 'memcached',
CACHE_LOCATION: 'localhost:11211',
CELERY_CONFIG_BROKER_HOST:'None',
CELERY_CONFIG_BROKER_PORT:0,
CELERY_CONFIG_BROKER_USER:'******',
CELERY_CONFIG_BROKER_PASSWORD:'******',
CELERY_CONFIG_BROKER_VHOST:'None',
CELERY_CONFIG_CELERY_RESULT_BACKEND:'None',
STATIC_ROOT: 'None',
REDIS_HOST:"localhost",
REDIS_PORT:6379,
HOSTNAME: "localhost",
PRIVATE_IP_ADDRESS: None,
PORT: 8000,
LOG_DIRECTORY: os.path.join(app_dir_path, "log"),
TIME_ZONE: "America/Chicago",
EMAIL_HOST: 'bar.com',
EMAIL_HOST_USER: "******",
EMAIL_FROM: '*****@*****.**',
EMAIL_HOST_PASSWORD: _gen_rand_string(6),
EMAIL_PORT: 25,
"_logging_fn": "%s.dummy_log_setup" % \
engage_settings_module,
"engage_django_components": lowered_components_list.__repr__()
}
else:
properties = deepcopy(properties)
properties["_logging_fn"] = "%s.log_setup_fn" % \
engage_settings_module
properties["engage_django_components"] = \
lowered_components_list.__repr__()
if properties["database_port"]==None:
properties["database_port"]=="''"
if properties[STATIC_ROOT]!=None:
properties[STATIC_ROOT] = '"' + properties[STATIC_ROOT] + '"'
else:
properties[STATIC_ROOT] = "None"
template = string.Template(engage_settings_template)
result = template.substitute(properties)
with open(engage_settings_py, "wb") as f:
f.write(result)
undo_ops.insert(0, (os.remove, [engage_settings_py]))
# now generate the deployed_settings.py file
template = string.Template(deployed_settings_template)
result = template.substitute({"django_settings_module": django_settings_module,
"engage_settings_module": engage_settings_module})
with open(deployed_settings_py, "wb") as f:
f.write(result)
undo_ops.insert(0, (os.remove, [deployed_settings_py]))
return undo_ops