def get_apps_from_cache():
try:
from django.apps import apps
return [app.models_module for app in apps.get_app_configs() if app.models_module]
except ImportError:
from django.db.models.loading import cache
return cache.get_apps()
def handle_noargs(self, **options):
if settings.MEDIA_ROOT == '':
print "MEDIA_ROOT is not set, nothing to do"
return
# Get a list of all files under MEDIA_ROOT
media = []
for root, dirs, files in os.walk(settings.MEDIA_ROOT):
for f in files:
media.append(os.path.abspath(os.path.join(root, f)))
# Get list of all fields (value) for each model (key)
# that is a FileField or subclass of a FileField
model_dict = defaultdict(list)
for app in cache.get_apps():
model_list = cache.get_models(app)
for model in model_list:
for field in model._meta.fields:
if issubclass(field.__class__, models.FileField):
model_dict[model].append(field)
# Get a list of all files referenced in the database
referenced = []
for model in model_dict.iterkeys():
all = model.objects.all().iterator()
for object in all:
for field in model_dict[model]:
referenced.append(
os.path.abspath(getattr(object, field.name).path))
# Print each file in MEDIA_ROOT that is not referenced in the database
for m in media:
if m not in referenced:
print m
def load_disk(self):
"""
Loads the migrations from all INSTALLED_APPS from disk.
"""
self.disk_migrations = {}
self.unmigrated_apps = set()
self.migrated_apps = set()
for app in cache.get_apps():
# Get the migrations module directory
app_label = app.__name__.split(".")[-2]
module_name = self.migrations_module(app_label)
was_loaded = module_name in sys.modules
try:
module = import_module(module_name)
except ImportError as e:
# I hate doing this, but I don't want to squash other import errors.
# Might be better to try a directory check directly.
if "No module named" in str(e) and "migrations" in str(e):
self.unmigrated_apps.add(app_label)
continue
raise
else:
# PY3 will happily import empty dirs as namespaces.
if not hasattr(module, '__file__'):
continue
# Module is not a package (e.g. migrations.py).
if not hasattr(module, '__path__'):
continue
# Force a reload if it's already loaded (tests need this)
if was_loaded:
six.moves.reload_module(module)
self.migrated_apps.add(app_label)
directory = os.path.dirname(module.__file__)
# Scan for .py[c|o] files
migration_names = set()
for name in os.listdir(directory):
if name.endswith(".py") or name.endswith(".pyc") or name.endswith(".pyo"):
import_name = name.rsplit(".", 1)[0]
if import_name[0] not in "_.~":
migration_names.add(import_name)
# Load them
south_style_migrations = False
for migration_name in migration_names:
try:
migration_module = import_module("%s.%s" % (module_name, migration_name))
except ImportError as e:
# Ignore South import errors, as we're triggering them
if "south" in str(e).lower():
south_style_migrations = True
break
raise
if not hasattr(migration_module, "Migration"):
raise BadMigrationError("Migration %s in app %s has no Migration class" % (migration_name, app_label))
# Ignore South-style migrations
if hasattr(migration_module.Migration, "forwards"):
south_style_migrations = True
break
self.disk_migrations[app_label, migration_name] = migration_module.Migration(migration_name, app_label)
if south_style_migrations:
self.unmigrated_apps.add(app_label)
def handle_noargs(self, **options):
if settings.MEDIA_ROOT == '':
print "MEDIA_ROOT is not set, nothing to do"
return
# Get a list of all files under MEDIA_ROOT
media = []
for root, dirs, files in os.walk(settings.MEDIA_ROOT):
for f in files:
media.append(os.path.abspath(os.path.join(root, f)))
# Get list of all fields (value) for each model (key)
# that is a FileField or subclass of a FileField
model_dict = defaultdict(list)
for app in cache.get_apps():
model_list = cache.get_models(app)
for model in model_list:
for field in model._meta.fields:
if issubclass(field.__class__, models.FileField):
model_dict[model].append(field)
# Get a list of all files referenced in the database
referenced = []
for model in model_dict.iterkeys():
all = model.objects.all().iterator()
for object in all:
for field in model_dict[model]:
referenced.append(os.path.abspath(getattr(object, field.name).path))
# Print each file in MEDIA_ROOT that is not referenced in the database
for m in media:
if m not in referenced:
print m
def get_apps_from_cache():
try:
from django.apps import apps
return [app.models_module for app in apps.get_app_configs() if app.models_module]
except ImportError:
from django.db.models.loading import cache
return cache.get_apps()
def load_disk(self):
"""
Loads the migrations from all INSTALLED_APPS from disk.
"""
self.disk_migrations = {}
self.unmigrated_apps = set()
self.migrated_apps = set()
for app in cache.get_apps():
# Get the migrations module directory
app_label = app.__name__.split(".")[-2]
module_name = self.migrations_module(app_label)
try:
module = import_module(module_name)
except ImportError as e:
# I hate doing this, but I don't want to squash other import errors.
# Might be better to try a directory check directly.
if "No module named" in str(e) and "migrations" in str(e):
self.unmigrated_apps.add(app_label)
continue
self.migrated_apps.add(app_label)
directory = os.path.dirname(module.__file__)
# Scan for .py[c|o] files
migration_names = set()
for name in os.listdir(directory):
if name.endswith(".py") or name.endswith(".pyc") or name.endswith(".pyo"):
import_name = name.rsplit(".", 1)[0]
if import_name[0] not in "_.~":
migration_names.add(import_name)
# Load them
for migration_name in migration_names:
migration_module = import_module("%s.%s" % (module_name, migration_name))
if not hasattr(migration_module, "Migration"):
raise BadMigrationError("Migration %s in app %s has no Migration class" % (migration_name, app_label))
self.disk_migrations[app_label, migration_name] = migration_module.Migration(migration_name, app_label)
def get_apps(self):
"""
Do not run Django own tests
"""
return filter(
lambda app: app.__name__.split('.', 1)[0] != 'django',
cache.get_apps())
def find_models_with_filefield():
for app in cache.get_apps():
model_list = cache.get_models(app)
for model in model_list:
for field in model._meta.fields:
if isinstance(field, models.FileField):
pre_save.connect(remove_old_files, sender=model)
post_delete.connect(remove_files, sender=model)
break
def StopServices(svc_list):
"""
Stop a set of services. Returns the list of those that
were stopped.
"""
old_path = []
old_environ = None
if not "DJANGO_SETTINGS_MODULE" in os.environ:
old_environ = True
os.environ["DJANGO_SETTINGS_MODULE"] = "freenasUI.settings"
if not "/usr/local/www" in sys.path:
old_path.append("/usr/local/www")
sys.path.append("/usr/local/www")
if not "/usr/local/www/freenasUI" in sys.path:
old_path.append("/usr/local/www/freenasUI")
sys.path.append("/usr/local/www/freenasUI")
from django.db.models.loading import cache
cache.get_apps()
from freenasUI.middleware.notifier import notifier
n = notifier()
retval = []
# Hm, this doesn't handle any particular ordering.
# May need to fix this.
for svc in svc_list:
if not svc in SERVICES:
raise ValueError("%s is not a known service" % svc)
s = SERVICES[svc]
svc_name = s["ServiceName"]
log.debug("StopServices: svc %s maps to %s" % (svc, svc_name))
if (not s["CheckStatus"]) or n.started(svc_name):
retval.append(svc)
n.stop(svc_name)
else:
log.debug("svc %s is not started" % svc)
# Should I remove the environment settings?
if old_environ:
os.environ.pop("DJANGO_SETTINGS_MODULE")
for p in old_path:
sys.path.remove(p)
return retval
def StopServices(svc_list):
"""
Stop a set of services. Returns the list of those that
were stopped.
"""
old_path = []
old_environ = None
if not "DJANGO_SETTINGS_MODULE" in os.environ:
old_environ = True
os.environ["DJANGO_SETTINGS_MODULE"] = "freenasUI.settings"
if not "/usr/local/www" in sys.path:
old_path.append("/usr/local/www")
sys.path.append("/usr/local/www")
if not "/usr/local/www/freenasUI" in sys.path:
old_path.append("/usr/local/www/freenasUI")
sys.path.append("/usr/local/www/freenasUI")
from django.db.models.loading import cache
cache.get_apps()
from freenasUI.middleware.notifier import notifier
n = notifier()
retval = []
# Hm, this doesn't handle any particular ordering.
# May need to fix this.
for svc in svc_list:
if not svc in SERVICES:
raise ValueError("%s is not a known service" % svc)
s = SERVICES[svc]
svc_name = s["ServiceName"]
log.debug("StopServices: svc %s maps to %s" % (svc, svc_name))
if (not s["CheckStatus"]) or n.started(svc_name):
retval.append(svc)
n.stop(svc_name)
else:
log.debug("svc %s is not started" % svc)
# Should I remove the environment settings?
if old_environ:
os.environ.pop("DJANGO_SETTINGS_MODULE")
for p in old_path:
sys.path.remove(p)
return retval
def find_models_with_filefield():
result = []
for app in cache.get_apps():
model_list = cache.get_models(app)
for model in model_list:
for field in model._meta.fields:
if isinstance(field, models.FileField):
result.append(model)
break
return result
def find_models_with_filefield():
result = []
for app in cache.get_apps():
model_list = cache.get_models(app)
for model in model_list:
for field in model._meta.fields:
if isinstance(field, models.FileField):
result.append(model)
break
return result
def _get_needed_files(self):
result = []
for app in cache.get_apps():
model_list = cache.get_models(app)
for model in model_list:
file_fields = [field.name for field in model._meta.fields
if field.get_internal_type() == 'FileField']
if len(file_fields) > 0:
files = model.objects.all().values_list(*file_fields)
result.extend([split_name(file)[0] for file in itertools.
chain.from_iterable(files) if file])
return result
def StartServices(svc_list):
"""
Start a set of services. THis is the output
from StopServices
"""
old_path = []
old_environ = None
if not "DJANGO_SETTINGS_MODULE" in os.environ:
old_environ = True
os.environ["DJANGO_SETTINGS_MODULE"] = "freenasUI.settings"
if not "/usr/local/www" in sys.path:
old_path.append("/usr/local/www")
sys.path.append("/usr/local/www")
if not "/usr/local/www/freenasUI" in sys.path:
old_path.append("/usr/local/www/freenasUI")
sys.path.append("/usr/local/www/freenasUI")
from django.db.models.loading import cache
cache.get_apps()
from freenasUI.middleware.notifier import notifier
n = notifier()
# Hm, this doesn't handle any particular ordering.
# May need to fix this.
for svc in svc_list:
if not svc in SERVICES:
raise ValueError("%s is not a known service" % svc)
svc_name = SERVICES[svc]["ServiceName"]
n.start(svc_name)
# Should I remove the environment settings?
if old_environ:
os.environ.pop("DJANGO_SETTINGS_MODULE")
for p in old_path:
sys.path.remove(p)
return
def StartServices(svc_list):
"""
Start a set of services. THis is the output
from StopServices
"""
old_path = []
old_environ = None
if not "DJANGO_SETTINGS_MODULE" in os.environ:
old_environ = True
os.environ["DJANGO_SETTINGS_MODULE"] = "freenasUI.settings"
if not "/usr/local/www" in sys.path:
old_path.append("/usr/local/www")
sys.path.append("/usr/local/www")
if not "/usr/local/www/freenasUI" in sys.path:
old_path.append("/usr/local/www/freenasUI")
sys.path.append("/usr/local/www/freenasUI")
from django.db.models.loading import cache
cache.get_apps()
from freenasUI.middleware.notifier import notifier
n = notifier()
# Hm, this doesn't handle any particular ordering.
# May need to fix this.
for svc in svc_list:
if not svc in SERVICES:
raise ValueError("%s is not a known service" % svc)
svc_name = SERVICES[svc]["ServiceName"]
n.start(svc_name)
# Should I remove the environment settings?
if old_environ:
os.environ.pop("DJANGO_SETTINGS_MODULE")
for p in old_path:
sys.path.remove(p)
return
def stashed_object_counts(request):
"""A context processor which adds counts of stashed objects
of models which inherit from SessionStashable to RequestContext.
To make a make a count appear for a particular model, set the
class attribute context_count_name to an appropriate string
to name its context variable, and enable this context processor
in settings.py.
"""
extra_context = {}
for app in cache.get_apps():
for model in cache.get_models(app):
if issubclass(model, SessionStashable) and model.context_count_name:
extra_context[model.context_count_name] = model.num_stashed_in_session(request.session)
return extra_context
def _get_app_model_modules(self):
"""Return the entire list of registered Django app models modules.
This is an internal utility function that's used to provide
compatibility with all supported versions of Django.
Returns:
list:
The list of Django applications.
"""
if apps:
return [
app.models_module for app in apps.get_app_configs()
if app.models_module
]
else:
return app_cache.get_apps()
def _get_app_model_modules(self):
"""Return the entire list of registered Django app models modules.
This is an internal utility function that's used to provide
compatibility with all supported versions of Django.
Returns:
list:
The list of Django applications.
"""
if apps:
return [
app.models_module
for app in apps.get_app_configs()
if app.models_module
]
else:
return app_cache.get_apps()
def get_apps():
"""Return the list of all installed apps with models.
This returns the apps from the app registry on Django >= 1.7, and from
the old-style cache on Django < 1.7.
Returns:
list: A list of all the modules containing model classes.
"""
if apps:
# Django >= 1.7
return [
app.models_module for app in apps.get_app_configs()
if app.models_module is not None
]
else:
# Django < 1.7
return cache.get_apps()
def _get_needed_files(self):
result = []
for app in cache.get_apps():
model_list = cache.get_models(app)
for model in model_list:
file_fields = [
field.name for field in model._meta.fields
if field.get_internal_type() == 'FileField'
]
if len(file_fields) > 0:
files = model.objects.all().values_list(*file_fields)
result.extend([
split_name(file)[0]
for file in itertools.chain.from_iterable(files)
if file
])
return result
def get_apps():
"""Return the list of all installed apps with models.
This returns the apps from the app registry on Django >= 1.7, and from
the old-style cache on Django < 1.7.
Returns:
list: A list of all the modules containing model classes.
"""
if apps:
# Django >= 1.7
return [
app.models_module
for app in apps.get_app_configs()
if app.models_module is not None
]
else:
# Django < 1.7
return cache.get_apps()
def handle_noargs(self, **options):
if settings.MEDIA_ROOT == '':
print("MEDIA_ROOT is not set, nothing to do")
return
# Get a list of all files under MEDIA_ROOT
media = []
for root, dirs, files in os.walk(settings.MEDIA_ROOT):
for f in files:
if ('geoserver_icons' not in root) and ('resized' not in root):
media.append(os.path.abspath(os.path.join(root, f)))
# Get list of all fields (value) for each model (key)
# that is a FileField or subclass of a FileField
model_dict = defaultdict(list)
for app in cache.get_apps():
model_list = cache.get_models(app)
for model in model_list:
for field in model._meta.fields:
if issubclass(field.__class__, models.FileField):
model_dict[model].append(field)
# Get a list of all files referenced in the database
referenced = []
for model in model_dict:
all = model.objects.all().iterator()
for object in all:
for field in model_dict[model]:
target_file = getattr(object, field.name)
if target_file:
referenced.append(os.path.abspath(target_file.path))
# Print each file in MEDIA_ROOT that is not referenced in the database
c = 0
for m in media:
if m not in referenced:
print 'Removing image %s' % m
os.remove(m)
c = c + 1
print 'Removed %s images, from a total of %s (referenced %s)' % (
c, len(media), len(referenced))
def find_orphaned_files(path=''):
"""Prints a list of all files in the path that are not referenced
in the database by all apps.
"""
if not getattr(settings, 'MEDIA_ROOT', None):
sys.stdout.write('MEDIA_ROOT is not set, nothing to do')
return
# Get a list of all files under MEDIA_ROOT.
media = set()
for root, dirs, files in os.walk(os.path.join(settings.MEDIA_ROOT, path)):
for f in files:
media.add(os.path.abspath(os.path.join(root, f)))
# Get list of all fields (value) for each model (key)
# that is a FileField or subclass of a FileField.
model_dict = defaultdict(list)
for app in cache.get_apps():
model_list = cache.get_models(app)
for model in model_list:
for field in model._meta.fields:
if issubclass(field.__class__, models.FileField):
model_dict[model].append(field)
# Get a list of all files referenced in the database.
referenced = set()
for model in model_dict.iterkeys():
all = model.objects.all().iterator()
for object in all:
for field in model_dict[model]:
f = getattr(object, field.name)
if f:
referenced.add(os.path.abspath(f.path))
# Print each file that is not referenced in the database.
for f in sorted(media - referenced):
sys.stdout.write(f)
sys.stdout.write('\n')
def find_orphaned_files(path=""):
"""Prints a list of all files in the path that are not referenced
in the database by all apps.
"""
if not getattr(settings, "MEDIA_ROOT", None):
sys.stdout.write("MEDIA_ROOT is not set, nothing to do")
return
# Get a list of all files under MEDIA_ROOT.
media = set()
for root, dirs, files in os.walk(os.path.join(settings.MEDIA_ROOT, path)):
for f in files:
media.add(os.path.abspath(os.path.join(root, f)))
# Get list of all fields (value) for each model (key)
# that is a FileField or subclass of a FileField.
model_dict = defaultdict(list)
for app in cache.get_apps():
model_list = cache.get_models(app)
for model in model_list:
for field in model._meta.fields:
if issubclass(field.__class__, models.FileField):
model_dict[model].append(field)
# Get a list of all files referenced in the database.
referenced = set()
for model in model_dict.iterkeys():
all = model.objects.all().iterator()
for object in all:
for field in model_dict[model]:
f = getattr(object, field.name)
if f:
referenced.add(os.path.abspath(f.path))
# Print each file that is not referenced in the database.
for f in sorted(media - referenced):
sys.stdout.write(f)
sys.stdout.write("\n")
import datetime
import logging
import os
import subprocess
import sys
sys.path.extend([
'/usr/local/www',
'/usr/local/www/freenasUI',
])
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'freenasUI.settings')
# Make sure to load all modules
from django.db.models.loading import cache
cache.get_apps()
from freenasUI.freeadmin.apppool import appPool
from freenasUI.storage.models import Replication
from freenasUI.common.timesubr import isTimeBetween
from freenasUI.common.pipesubr import pipeopen, system
from freenasUI.common.locks import mntlock
from freenasUI.common.system import send_mail
# DESIGN NOTES
#
# A snapshot transists its state in its lifetime this way:
# NEW: A newly created snapshot by autosnap
# LATEST: A snapshot marked to be the latest one
# -: The replication system no longer cares this.
#
import tdb
import tempfile
import time
import logging
from dns import resolver
sys.path.extend([
'/usr/local/www',
'/usr/local/www/freenasUI'
])
os.environ["DJANGO_SETTINGS_MODULE"] = "freenasUI.settings"
# Make sure to load all modules
from django.db.models.loading import cache
cache.get_apps()
from django.db.models import Q
from freenasUI.account.models import (
bsdUsers,
bsdGroups,
bsdGroupMembership
)
from freenasUI.common.freenasldap import (
FreeNAS_ActiveDirectory,
FLAGS_DBINIT
)
from freenasUI.common.pipesubr import pipeopen
from freenasUI.common.log import log_traceback
from freenasUI.common.samba import Samba4
def all_concrete_models():
return [(app, [model for model in cache.get_models(app) if not model._meta.abstract]) for app in cache.get_apps() if cache.get_models(app)]
def _stashable_models():
for app in cache.get_apps():
for model in cache.get_models(app):
if issubclass(model, SessionStashable):
yield model
def _stashable_models():
for app in cache.get_apps():
for model in cache.get_models(app):
if issubclass(model, SessionStashable):
yield model
