def get_slaves(self, domain):
ips = []
masters_ips = self.get_masters_ips(domain)
records = domain.get_records()
# Slaves from NS
for record in records.by_type(Record.NS):
hostname = record.value.rstrip('.')
# First try with a DNS query, a more reliable source
try:
addr = socket.gethostbyname(hostname)
except socket.gaierror:
# check if hostname is declared
try:
domain = Domain.objects.get(name=hostname)
except Domain.DoesNotExist:
continue
else:
# default to domain A record address
addr = records.by_type(Record.A)[0].value
if addr not in masters_ips:
ips.append(addr)
# Slaves from internal networks
if not settings.DOMAINS_MASTERS:
for server in self.get_servers(domain, Bind9SlaveDomainController):
ips.append(server)
return OrderedSet(sorted(ips))
def get_pending_operations(cls):
# Check if an error poped up before OperationsMiddleware.process_request()
if hasattr(cls.thread_locals, 'request'):
request = cls.thread_locals.request
if not hasattr(request, 'pending_operations'):
request.pending_operations = OrderedSet()
return request.pending_operations
return set()
def get_php_init_vars(self, merge=settings.WEBAPPS_MERGE_PHP_WEBAPPS):
""" Prepares PHP options for inclusion on php.ini """
init_vars = OrderedDict()
options = self.get_options(merge=merge)
php_version_number = float(self.get_php_version_number())
timeout = None
for name, value in options.items():
if name == 'timeout':
timeout = value
else:
opt = AppOption.get(name)
# Filter non-deprecated PHP options
if opt.group == opt.PHP and (opt.deprecated
or 999) > php_version_number:
init_vars[name] = value
# Disable functions
if self.PHP_DISABLED_FUNCTIONS:
enable_functions = init_vars.pop('enable_functions', None)
enable_functions = OrderedSet(
enable_functions.split(',') if enable_functions else ())
disable_functions = init_vars.pop('disable_functions', None)
disable_functions = OrderedSet(
disable_functions.split(',') if disable_functions else ())
if disable_functions or enable_functions or self.is_fpm:
# FPM: Defining 'disable_functions' or 'disable_classes' will not overwrite previously
# defined php.ini values, but will append the new value
for function in self.PHP_DISABLED_FUNCTIONS:
if function not in enable_functions:
disable_functions.add(function)
init_vars['disable_functions'] = ','.join(disable_functions)
# Process timeout
if timeout:
timeout = max(settings.WEBAPPS_PYTHON_DEFAULT_TIMEOUT,
int(timeout))
# Give a little slack here
timeout = str(timeout - 2)
init_vars['max_execution_time'] = timeout
# Custom error log
if self.PHP_ERROR_LOG_PATH and 'error_log' not in init_vars:
context = self.get_directive_context()
error_log_path = os.path.normpath(self.PHP_ERROR_LOG_PATH %
context)
init_vars['error_log'] = error_log_path
# Auto update max_post_size
if 'upload_max_filesize' in init_vars:
upload_max_filesize = init_vars['upload_max_filesize']
post_max_size = init_vars.get('post_max_size', '0')
upload_max_filesize_value = eval(
upload_max_filesize.replace('M', '*1024'))
post_max_size_value = eval(post_max_size.replace('M', '*1024'))
init_vars['post_max_size'] = post_max_size
if upload_max_filesize_value > post_max_size_value:
init_vars['post_max_size'] = upload_max_filesize
return init_vars
def get_php_init_vars(self, merge=settings.WEBAPPS_MERGE_PHP_WEBAPPS):
""" Prepares PHP options for inclusion on php.ini """
init_vars = OrderedDict()
options = self.get_options(merge=merge)
php_version_number = float(self.get_php_version_number())
timeout = None
for name, value in options.items():
if name == 'timeout':
timeout = value
else:
opt = AppOption.get(name)
# Filter non-deprecated PHP options
if opt.group == opt.PHP and (opt.deprecated or 999) > php_version_number:
init_vars[name] = value
# Disable functions
if self.PHP_DISABLED_FUNCTIONS:
enable_functions = init_vars.pop('enable_functions', None)
enable_functions = OrderedSet(enable_functions.split(',') if enable_functions else ())
disable_functions = init_vars.pop('disable_functions', None)
disable_functions = OrderedSet(disable_functions.split(',') if disable_functions else ())
if disable_functions or enable_functions or self.is_fpm:
# FPM: Defining 'disable_functions' or 'disable_classes' will not overwrite previously
# defined php.ini values, but will append the new value
for function in self.PHP_DISABLED_FUNCTIONS:
if function not in enable_functions:
disable_functions.add(function)
init_vars['disable_functions'] = ','.join(disable_functions)
# Process timeout
if timeout:
timeout = max(settings.WEBAPPS_PYTHON_DEFAULT_TIMEOUT, int(timeout))
# Give a little slack here
timeout = str(timeout-2)
init_vars['max_execution_time'] = timeout
# Custom error log
if self.PHP_ERROR_LOG_PATH and 'error_log' not in init_vars:
context = self.get_directive_context()
error_log_path = os.path.normpath(self.PHP_ERROR_LOG_PATH % context)
init_vars['error_log'] = error_log_path
# Auto update max_post_size
if 'upload_max_filesize' in init_vars:
upload_max_filesize = init_vars['upload_max_filesize']
post_max_size = init_vars.get('post_max_size', '0')
upload_max_filesize_value = eval(upload_max_filesize.replace('M', '*1024'))
post_max_size_value = eval(post_max_size.replace('M', '*1024'))
init_vars['post_max_size'] = post_max_size
if upload_max_filesize_value > post_max_size_value:
init_vars['post_max_size'] = upload_max_filesize
return init_vars
def handle(self, *args, **options):
list_backends = options.get('list_backends')
if list_backends:
for backend in ServiceBackend.get_backends():
self.stdout.write(str(backend).split("'")[1])
return
model = apps.get_model(*options['model'].split('.'))
action = options.get('action')
interactive = options.get('interactive')
servers = options.get('servers')
backends = options.get('backends')
if (servers and not backends) or (not servers and backends):
raise CommandError("--backends and --servers go in tandem.")
dry = options.get('dry')
kwargs = {}
for comp in options.get('query', []):
comps = iter(comp.split('='))
for arg in comps:
kwargs[arg] = next(comps).strip().rstrip(',')
operations = OrderedSet()
route_cache = {}
queryset = model.objects.filter(**kwargs).order_by('id')
if servers:
servers = servers.split(',')
backends = backends.split(',')
server_objects = []
# Get and create missing Servers
for server in servers:
try:
server = Server.objects.get(address=server)
except Server.DoesNotExist:
server = Server(name=server, address=server)
server.full_clean()
server.save()
server_objects.append(server)
# Generate operations for the given backend
for instance in queryset:
for backend in backends:
backend = import_class(backend)
operations.add(Operation(backend, instance, action, servers=server_objects))
else:
for instance in queryset:
manager.collect(instance, action, operations=operations, route_cache=route_cache)
scripts, block = manager.generate(operations)
servers = []
# Print scripts
for key, value in scripts.items():
server, __ = key
backend, operations = value
servers.append(server.name)
self.stdout.write('# Execute on %s' % server.name)
for method, commands in backend.scripts:
script = '\n'.join(commands)
self.stdout.write(script)
if interactive:
context = {
'servers': ', '.join(servers),
}
msg = ("\n\nAre your sure to execute the previous scripts on %(servers)s (yes/no)? " % context)
confirm = input(msg)
while 1:
if confirm not in ('yes', 'no'):
confirm = input('Please enter either "yes" or "no": ')
continue
if confirm == 'no':
return
break
if not dry:
logs = manager.execute(scripts, block=block)
for log in logs:
self.stdout.write(log.stdout)
self.stderr.write(log.stderr)
for log in logs:
self.stdout.write(' '.join((log.backend, log.state)))
def get_masters_ips(self, domain):
ips = list(settings.DOMAINS_MASTERS)
if not ips:
ips += self.get_servers(domain, Bind9MasterDomainController)
return OrderedSet(sorted(ips))
def collect(instance, action, **kwargs):
""" collect operations """
operations = kwargs.get('operations', OrderedSet())
route_cache = kwargs.get('route_cache', {})
for backend_cls in ServiceBackend.get_backends():
# Check if there exists a related instance to be executed for this backend and action
instances = []
if action in backend_cls.actions:
if backend_cls.is_main(instance):
instances = [(instance, action)]
else:
for candidate in backend_cls.get_related(instance):
if candidate.__class__.__name__ == 'ManyRelatedManager':
if 'pk_set' in kwargs:
# m2m_changed signal
candidates = kwargs['model'].objects.filter(
pk__in=kwargs['pk_set'])
else:
candidates = candidate.all()
else:
candidates = [candidate]
for candidate in candidates:
# Check if a delete for candidate is in operations
delete_mock = Operation(backend_cls, candidate,
Operation.DELETE)
if delete_mock not in operations:
# related objects with backend.model trigger save()
instances.append((candidate, Operation.SAVE))
for selected, iaction in instances:
# Maintain consistent state of operations based on save/delete behaviour
# Prevent creating a deleted selected by deleting existing saves
if iaction == Operation.DELETE:
save_mock = Operation(backend_cls, selected, Operation.SAVE)
try:
operations.remove(save_mock)
except KeyError:
pass
else:
update_fields = kwargs.get('update_fields', None)
if update_fields is not None:
# TODO remove this, django does not execute post_save if update_fields=[]...
# Maybe open a ticket at Djangoproject ?
# INITIAL INTENTION: "update_fields=[]" is a convention for explicitly executing backend
# i.e. account.disable()
if update_fields != []:
execute = False
for field in update_fields:
if field not in backend_cls.ignore_fields:
execute = True
break
if not execute:
continue
operation = Operation(backend_cls, selected, iaction)
# Only schedule operations if the router has execution routes
routes = router.objects.get_for_operation(operation,
cache=route_cache)
if routes:
logger.debug("Operation %s collected for execution" %
operation)
operation.routes = routes
if iaction != Operation.DELETE:
# usually we expect to be using last object state,
# except when we are deleting it
operations.discard(operation)
elif iaction == Operation.DELETE:
operation.preload_context()
operations.add(operation)
return operations
def __enter__(self):
cls = type(self)
self.old_pending_operations = cls.thread_locals.pending_operations
cls.thread_locals.pending_operations = OrderedSet()
self.old_route_cache = cls.thread_locals.route_cache
cls.thread_locals.route_cache = {}
def handle(self, *args, **options):
list_backends = options.get('list_backends')
if list_backends:
for backend in ServiceBackend.get_backends():
self.stdout.write(str(backend).split("'")[1])
return
model = apps.get_model(*options['model'].split('.'))
action = options.get('action')
interactive = options.get('interactive')
servers = options.get('servers')
backends = options.get('backends')
if (servers and not backends) or (not servers and backends):
raise CommandError("--backends and --servers go in tandem.")
dry = options.get('dry')
kwargs = {}
for comp in options.get('query', []):
comps = iter(comp.split('='))
for arg in comps:
kwargs[arg] = next(comps).strip().rstrip(',')
operations = OrderedSet()
route_cache = {}
queryset = model.objects.filter(**kwargs).order_by('id')
if servers:
servers = servers.split(',')
backends = backends.split(',')
routes = []
# Get and create missing Servers
for server in servers:
try:
server = Server.objects.get(address=server)
except Server.DoesNotExist:
server = Server(name=server, address=server)
server.full_clean()
server.save()
routes.append(AttrDict(
host=server,
async=False,
action_is_async=lambda self: False,
))
# Generate operations for the given backend
for instance in queryset:
for backend in backends:
backend = import_class(backend)
operations.add(Operation(backend, instance, action, routes=routes))
else:
for instance in queryset:
manager.collect(instance, action, operations=operations, route_cache=route_cache)
scripts, serialize = manager.generate(operations)
servers = []
# Print scripts
for key, value in scripts.items():
route, __, __ = key
backend, operations = value
servers.append(str(route.host))
self.stdout.write('# Execute %s on %s' % (backend.get_name(), route.host))
for method, commands in backend.scripts:
script = '\n'.join(commands)
self.stdout.write(script)
if interactive:
context = {
'servers': ', '.join(servers),
}
if not confirm("\n\nAre your sure to execute the previous scripts on %(servers)s (yes/no)? " % context):
return
if not dry:
logs = manager.execute(scripts, serialize=serialize, async=True)
running = list(logs)
stdout = 0
stderr = 0
while running:
for log in running:
cstdout = len(log.stdout)
cstderr = len(log.stderr)
if cstdout > stdout:
self.stdout.write(log.stdout[stdout:])
stdout = cstdout
if cstderr > stderr:
self.stderr.write(log.stderr[stderr:])
stderr = cstderr
if log.has_finished:
running.remove(log)
time.sleep(0.05)
for log in logs:
self.stdout.write(' '.join((log.backend, log.state)))
def collect_operations(self, **options):
model = options.get('model')
backends = options.get('backends') or set()
if backends:
backends = set(backends.split(','))
servers = options.get('servers') or set()
if servers:
servers = set([
Server.objects.get(Q(address=server) | Q(name=server))
for server in servers.split(',')
])
action = options.get('action')
if not model:
models = set()
if servers:
for server in servers:
if backends:
routes = server.routes.filter(backend__in=backends)
else:
routes = server.routes.all()
elif backends:
routes = Route.objects.filter(backend__in=backends)
else:
raise CommandError("Model or --servers or --backends?")
for route in routes.filter(is_active=True):
model = route.backend_class.model_class()
models.add(model)
querysets = [model.objects.order_by('id') for model in models]
else:
kwargs = {}
for comp in options.get('query', []):
comps = iter(comp.split('='))
for arg in comps:
kwargs[arg] = next(comps).strip().rstrip(',')
model = apps.get_model(*model.split('.'))
queryset = model.objects.filter(**kwargs).order_by('id')
querysets = [queryset]
operations = OrderedSet()
route_cache = {}
for queryset in querysets:
for instance in queryset:
manager.collect(instance,
action,
operations=operations,
route_cache=route_cache)
if backends:
result = []
for operation in operations:
if operation.backend in backends:
result.append(operation)
operations = result
if servers:
routes = []
result = []
for operation in operations:
routes = [
route for route in operation.routes
if route.host in servers
]
operation.routes = routes
if routes:
result.append(operation)
operations = result
return operations
def orchestrate(modeladmin, request, queryset):
operations = set()
action = Operation.SAVE
operations = OrderedSet()
if queryset.model is Route:
for route in queryset:
routes = [route]
backend = route.backend_class
if action not in backend.actions:
continue
for instance in backend.model_class().objects.all():
if route.matches(instance):
operations.add(Operation(backend, instance, action, routes=routes))
elif queryset.model is Server:
models = set()
for server in queryset:
routes = server.routes.all()
for route in routes.filter(is_active=True):
model = route.backend_class.model_class()
models.add(model)
querysets = [model.objects.order_by('id') for model in models]
route_cache = {}
for model in models:
for instance in model.objects.all():
manager.collect(instance, action, operations=operations, route_cache=route_cache)
routes = []
result = []
for operation in operations:
routes = [route for route in operation.routes if route.host in queryset]
operation.routes = routes
if routes:
result.append(operation)
operations = result
if not operations:
messages.warning(request, _("No related operations."))
return
if request.POST.get('post') == 'generic_confirmation':
logs = Operation.execute(operations)
message_user(request, logs)
for obj in queryset:
modeladmin.log_change(request, obj, 'Orchestrated')
return
opts = modeladmin.model._meta
display_objects = {}
for operation in operations:
try:
display_objects[operation.backend].append(operation)
except KeyError:
display_objects[operation.backend] = [operation]
context = {
'title': _("Are you sure to execute the following operations?"),
'action_name': _('Orchestrate'),
'action_value': 'orchestrate',
'display_objects': display_objects,
'queryset': queryset,
'opts': opts,
'app_label': opts.app_label,
'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME,
'obj': get_object_from_url(modeladmin, request),
}
return render(request, 'admin/orchestration/orchestrate.html', context)
