def validate_import_class(cls, value):
try:
import_class(value)
except ImportError as exc:
if "cannot import name 'settings'" in str(exc):
# circular dependency on init time
pass
except Exception as exc:
raise ValidationError(format_exception(exc))
def clean_identifier(self, service=service):
identifier = self.cleaned_data['identifier']
validator_path = settings.MISCELLANEOUS_IDENTIFIER_VALIDATORS.get(service.name, None)
if validator_path:
validator = import_class(validator_path)
validator(identifier)
return identifier
def clean_identifier(self, service=service):
identifier = self.cleaned_data['identifier']
validator_path = settings.MISCELLANEOUS_IDENTIFIER_VALIDATORS.get(service.name, None)
if validator_path:
validator = import_class(validator_path)
validator(identifier)
return identifier
def get_plugins(cls, all=False):
if all:
plugins = super().get_plugins()
else:
plugins = []
for cls in settings.WEBAPPS_ENABLED_OPTIONS:
plugins.append(import_class(cls))
return plugins
def get_plugins(cls):
plugins = []
for cls in settings.PAYMENTS_ENABLED_METHODS:
try:
plugins.append(import_class(cls))
except ImportError as exc:
logger.error('Error loading %s: %s' % (cls, str(exc)))
return plugins
def get_plugins(cls, all=False):
if all:
plugins = super().get_plugins()
else:
plugins = []
for cls in settings.WEBSITES_ENABLED_DIRECTIVES:
plugins.append(import_class(cls))
return plugins
def get_plugins(cls, all=False):
if all:
plugins = super().get_plugins()
else:
plugins = []
for cls in settings.WEBSITES_ENABLED_DIRECTIVES:
plugins.append(import_class(cls))
return plugins
def get_plugins(cls, all=False):
if all:
plugins = super().get_plugins()
else:
plugins = []
for cls in settings.WEBAPPS_ENABLED_OPTIONS:
plugins.append(import_class(cls))
return plugins
def get_plugins(cls, all=False):
if all:
for module in os.listdir(os.path.dirname(__file__)):
if module != "__init__.py" and module[-3:] == ".py":
importlib.import_module("." + module[:-3], __package__)
plugins = super().get_plugins()
else:
plugins = []
for cls in settings.WEBAPPS_TYPES:
plugins.append(import_class(cls))
return plugins
def get_plugins(cls, all=False):
if all:
for module in os.listdir(os.path.dirname(__file__)):
if module not in ('options.py', '__init__.py') and module[-3:] == '.py':
importlib.import_module('.'+module[:-3], __package__)
plugins = super().get_plugins()
else:
plugins = []
for cls in settings.SAAS_ENABLED_SERVICES:
plugins.append(import_class(cls))
return plugins
def get_plugins(cls, all=False):
if all:
for module in os.listdir(os.path.dirname(__file__)):
if module not in ('options.py', '__init__.py') and module[-3:] == '.py':
importlib.import_module('.'+module[:-3], __package__)
plugins = super().get_plugins()
else:
plugins = []
for cls in settings.PAYMENTS_ENABLED_METHODS:
plugins.append(import_class(cls))
return plugins
def get_plugins(cls, all=False):
if all:
for module in os.listdir(os.path.dirname(__file__)):
if module != '__init__.py' and module[-3:] == '.py':
importlib.import_module('.' + module[:-3], __package__)
plugins = super().get_plugins()
else:
plugins = []
for cls in settings.WEBAPPS_TYPES:
plugins.append(import_class(cls))
return plugins
def handle(self, *args, **options):
method = import_class(options['method'])
kwargs = {}
arguments = []
for arg in args:
if '=' in args:
name, value = arg.split('=')
if value.isdigit():
value = int(value)
kwargs[name] = value
else:
if arg.isdigit():
arg = int(arg)
arguments.append(arg)
args = arguments
keep_state(method)(get_id(), get_name(method), *args, **kwargs)
def handle(self, *args, **options):
method = import_class(options["method"])
kwargs = {}
arguments = []
for arg in args:
if "=" in args:
name, value = arg.split("=")
if value.isdigit():
value = int(value)
kwargs[name] = value
else:
if arg.isdigit():
arg = int(arg)
arguments.append(arg)
args = arguments
keep_state(method)(get_id(), get_name(method), *args, **kwargs)
def execute(operations):
""" generates and executes the operations on the servers """
router = import_class(settings.ORCHESTRATION_ROUTER)
# Generate scripts per server+backend
scripts = {}
cache = {}
for operation in operations:
servers = router.get_servers(operation, cache=cache)
for server in servers:
key = (server, operation.backend)
if key not in scripts:
scripts[key] = (operation.backend(), [operation])
scripts[key][0].prepare()
else:
scripts[key][1].append(operation)
method = getattr(scripts[key][0], operation.action)
method(operation.instance)
# Execute scripts on each server
threads = []
executions = []
for key, value in scripts.iteritems():
server, __ = key
backend, operations = value
backend.commit()
execute = as_task(backend.execute)
execute = close_connection(execute)
thread = threading.Thread(target=execute, args=(server,))
thread.start()
threads.append(thread)
executions.append((execute, operations))
[ thread.join() for thread in threads ]
logs = []
for execution, operations in executions:
for operation in operations:
operation.log = execution.log
operation.save()
logs.append(execution.log)
return logs
def SSH(*args, **kwargs):
""" facade function enabling to chose between multiple SSH backends"""
method = import_class(settings.ORCHESTRATION_SSH_METHOD_BACKEND)
return method(*args, **kwargs)
def SSH(*args, **kwargs):
""" facade function enabling to chose between multiple SSH backends"""
method = import_class(settings.ORCHESTRATION_SSH_METHOD_BACKEND)
return method(*args, **kwargs)
from django.db import models
from django.apps import apps
from django.utils.functional import cached_property
from django.utils.module_loading import autodiscover_modules
from django.utils.translation import ugettext_lazy as _
from orchestra.core import caches, validators
from orchestra.utils.python import import_class
from . import settings
from .handlers import ServiceHandler
from .helpers import get_rate_methods_help_text
autodiscover_modules('handlers')
rate_class = import_class(settings.SERVICES_RATE_CLASS)
class ServiceQuerySet(models.QuerySet):
def filter_by_instance(self, instance):
cache = caches.get_request_cache()
ct = ContentType.objects.get_for_model(instance)
key = 'services.Service-%i' % ct.pk
services = cache.get(key)
if services is None:
services = self.filter(content_type=ct, is_active=True)
cache.set(key, services)
return services
class Service(models.Model):
def validate_import_class(cls, value):
try:
import_class(value)
except Exception as exc:
raise ValidationError(format_exception(exc))
def validate_import_class(cls, value):
try:
import_class(value)
except Exception as exc:
raise ValidationError(format_exception(exc))
def handle(self, *args, **options):
list_backends = options.get('list_backends')
if list_backends:
for backend in ServiceBackend.get_backends():
self.stdout.write(str(backend).split("'")[1])
return
model = apps.get_model(*options['model'].split('.'))
action = options.get('action')
interactive = options.get('interactive')
servers = options.get('servers')
backends = options.get('backends')
if (servers and not backends) or (not servers and backends):
raise CommandError("--backends and --servers go in tandem.")
dry = options.get('dry')
kwargs = {}
for comp in options.get('query', []):
comps = iter(comp.split('='))
for arg in comps:
kwargs[arg] = next(comps).strip().rstrip(',')
operations = OrderedSet()
route_cache = {}
queryset = model.objects.filter(**kwargs).order_by('id')
if servers:
servers = servers.split(',')
backends = backends.split(',')
routes = []
# Get and create missing Servers
for server in servers:
try:
server = Server.objects.get(address=server)
except Server.DoesNotExist:
server = Server(name=server, address=server)
server.full_clean()
server.save()
routes.append(AttrDict(
host=server,
async=False,
action_is_async=lambda self: False,
))
# Generate operations for the given backend
for instance in queryset:
for backend in backends:
backend = import_class(backend)
operations.add(Operation(backend, instance, action, routes=routes))
else:
for instance in queryset:
manager.collect(instance, action, operations=operations, route_cache=route_cache)
scripts, serialize = manager.generate(operations)
servers = []
# Print scripts
for key, value in scripts.items():
route, __, __ = key
backend, operations = value
servers.append(str(route.host))
self.stdout.write('# Execute %s on %s' % (backend.get_name(), route.host))
for method, commands in backend.scripts:
script = '\n'.join(commands)
self.stdout.write(script)
if interactive:
context = {
'servers': ', '.join(servers),
}
if not confirm("\n\nAre your sure to execute the previous scripts on %(servers)s (yes/no)? " % context):
return
if not dry:
logs = manager.execute(scripts, serialize=serialize, async=True)
running = list(logs)
stdout = 0
stderr = 0
while running:
for log in running:
cstdout = len(log.stdout)
cstderr = len(log.stderr)
if cstdout > stdout:
self.stdout.write(log.stdout[stdout:])
stdout = cstdout
if cstderr > stderr:
self.stderr.write(log.stderr[stderr:])
stderr = cstderr
if log.has_finished:
running.remove(log)
time.sleep(0.05)
for log in logs:
self.stdout.write(' '.join((log.backend, log.state)))
def handle(self, *args, **options):
list_backends = options.get('list_backends')
if list_backends:
for backend in ServiceBackend.get_backends():
self.stdout.write(str(backend).split("'")[1])
return
model = apps.get_model(*options['model'].split('.'))
action = options.get('action')
interactive = options.get('interactive')
servers = options.get('servers')
backends = options.get('backends')
if (servers and not backends) or (not servers and backends):
raise CommandError("--backends and --servers go in tandem.")
dry = options.get('dry')
kwargs = {}
for comp in options.get('query', []):
comps = iter(comp.split('='))
for arg in comps:
kwargs[arg] = next(comps).strip().rstrip(',')
operations = OrderedSet()
route_cache = {}
queryset = model.objects.filter(**kwargs).order_by('id')
if servers:
servers = servers.split(',')
backends = backends.split(',')
server_objects = []
# Get and create missing Servers
for server in servers:
try:
server = Server.objects.get(address=server)
except Server.DoesNotExist:
server = Server(name=server, address=server)
server.full_clean()
server.save()
server_objects.append(server)
# Generate operations for the given backend
for instance in queryset:
for backend in backends:
backend = import_class(backend)
operations.add(Operation(backend, instance, action, servers=server_objects))
else:
for instance in queryset:
manager.collect(instance, action, operations=operations, route_cache=route_cache)
scripts, block = manager.generate(operations)
servers = []
# Print scripts
for key, value in scripts.items():
server, __ = key
backend, operations = value
servers.append(server.name)
self.stdout.write('# Execute on %s' % server.name)
for method, commands in backend.scripts:
script = '\n'.join(commands)
self.stdout.write(script)
if interactive:
context = {
'servers': ', '.join(servers),
}
msg = ("\n\nAre your sure to execute the previous scripts on %(servers)s (yes/no)? " % context)
confirm = input(msg)
while 1:
if confirm not in ('yes', 'no'):
confirm = input('Please enter either "yes" or "no": ')
continue
if confirm == 'no':
return
break
if not dry:
logs = manager.execute(scripts, block=block)
for log in logs:
self.stdout.write(log.stdout)
self.stderr.write(log.stderr)
for log in logs:
self.stdout.write(' '.join((log.backend, log.state)))
import traceback
from collections import OrderedDict
from django.core.mail import mail_admins
from orchestra.utils import db
from orchestra.utils.python import import_class, OrderedSet
from . import settings, Operation
from .backends import ServiceBackend
from .helpers import send_report
from .models import BackendLog
from .signals import pre_action, post_action, pre_commit, post_commit, pre_prepare, post_prepare
logger = logging.getLogger(__name__)
router = import_class(settings.ORCHESTRATION_ROUTER)
def keep_log(execute, log, operations):
def wrapper(*args, **kwargs):
""" send report """
# Remember that threads have their oun connection poll
# No need to EVER temper with the transaction here
log = kwargs['log']
try:
log = execute(*args, **kwargs)
except Exception as e:
trace = traceback.format_exc()
log.state = log.EXCEPTION
log.stderr += trace
log.save()
def get_bill_backend(cls):
return import_class(settings.ORDERS_BILLING_BACKEND)()
from collections import OrderedDict
from django import db
from django.core.mail import mail_admins
from orchestra.utils.python import import_class, OrderedSet
from . import settings, Operation
from .backends import ServiceBackend
from .helpers import send_report
from .models import BackendLog
from .signals import pre_action, post_action
logger = logging.getLogger(__name__)
router = import_class(settings.ORCHESTRATION_ROUTER)
def as_task(execute):
def wrapper(*args, **kwargs):
""" send report """
# Tasks run on a separate transaction pool (thread), no need to temper with the transaction
try:
log = execute(*args, **kwargs)
if log.state != log.SUCCESS:
send_report(execute, args, log)
except Exception as e:
subject = 'EXCEPTION executing backend(s) %s %s' % (str(args), str(kwargs))
message = traceback.format_exc()
logger.error(subject)
logger.error(message)
def get_api_root_view(self):
""" returns the root view, with all the linked collections """
APIRoot = import_class(settings.ORCHESTRA_API_ROOT_VIEW)
APIRoot.router = self
return APIRoot.as_view()
def get_plugins(cls):
plugins = []
for cls in settings.PAYMENTS_ENABLED_METHODS:
plugins.append(import_class(cls))
return plugins
def get_plugins(cls):
plugins = []
for cls in settings.WEBAPPS_TYPES:
plugins.append(import_class(cls))
return plugins
def get_bill_backend(cls):
return import_class(settings.ORDERS_BILLING_BACKEND)()
def get_methods(cls):
return dict((method, import_class(method))
for method in settings.PLANS_RATE_METHODS)
def get_plugins(cls):
plugins = []
for cls in settings.WEBAPPS_ENABLED_OPTIONS:
plugins.append(import_class(cls))
return plugins
def get_methods(cls):
return dict((method, import_class(method)) for method in settings.PLANS_RATE_METHODS)
def get_plugins(cls):
plugins = []
for cls in settings.WEBSITES_ENABLED_DIRECTIVES:
plugins.append(import_class(cls))
return plugins
def get_api_root_view(self, api_urls=None):
""" returns the root view, with all the linked collections """
APIRoot = import_class(settings.ORCHESTRA_API_ROOT_VIEW)
APIRoot.router = self
return APIRoot.as_view()
def get_plugins(cls):
plugins = []
for cls in settings.SAAS_ENABLED_SERVICES:
plugins.append(import_class(cls))
return plugins
from django.db import models
from django.apps import apps
from django.utils.functional import cached_property
from django.utils.module_loading import autodiscover_modules
from django.utils.translation import string_concat, ugettext_lazy as _
from orchestra.core import caches, validators
from orchestra.utils.python import import_class
from . import settings
from .handlers import ServiceHandler
autodiscover_modules('handlers')
rate_class = import_class(settings.SERVICES_RATE_CLASS)
class ServiceQuerySet(models.QuerySet):
def filter_by_instance(self, instance):
cache = caches.get_request_cache()
ct = ContentType.objects.get_for_model(instance)
key = 'services.Service-%i' % ct.pk
services = cache.get(key)
if services is None:
services = self.filter(content_type=ct, is_active=True)
cache.set(key, services)
return services
class Service(models.Model):
