def __init__(self,
wait=2,
threshold=4,
stdout=None,
stderr=None,
no_color=False):
"""Task Daemon: Gets task and execute them.
:arg wait : How many seconds should wait before check new records again. This should be lower than threshold
:arg threshold :
"""
self.wait = wait
self.threshold = threshold
self.threads = []
self.queues = []
self.stdout = OutputWrapper(stdout or sys.stdout)
self.stderr = OutputWrapper(stderr or sys.stderr)
if no_color:
self.style = no_style()
else:
self.style = color_style()
self.stderr.style_func = self.style.ERROR
super(Daemon, self).__init__(name="Daemon",
pidfile=_pidfile,
runfile=_runfile,
stoptimeout=10,
debug=1)
def run_on_schema(
schema_name, executor_codename, command_class, function_name=None, args=[], kwargs={}, pass_schema_in_kwargs=False
):
style = color.color_style()
stdout = OutputWrapper(sys.stdout)
stderr = OutputWrapper(sys.stderr)
stdout.style_func = stderr.style_func = lambda message: "[%s:%s] %s" % (
style.NOTICE(executor_codename),
style.NOTICE(schema_name),
message,
)
command = command_class(stdout=stdout, stderr=stderr)
connections.close_all()
connection.set_schema_to(schema_name)
if pass_schema_in_kwargs:
kwargs.update({"schema_name": schema_name})
if function_name == "special:call_command":
call_command(command, *args, **kwargs)
elif function_name == "special:run_from_argv":
command.run_from_argv(args)
else:
getattr(command, function_name)(*args, **kwargs)
transaction.commit()
connection.close()
def execute(self, *args, **options):
"""
Try to execute this command, performing system checks if needed (as
controlled by the ``requires_system_checks`` attribute, except if
force-skipped).
"""
if options['no_color']:
self.style = no_style()
self.stderr.style_func = None
if options.get('stdout'):
self.stdout = OutputWrapper(options['stdout'])
if options.get('stderr'):
self.stderr = OutputWrapper(options['stderr'],
self.stderr.style_func)
if self.requires_system_checks and not options.get('skip_checks'):
self.check()
if self.requires_migrations_checks:
self.check_migrations()
output = self.handle(*args, **options)
if output:
if self.output_transaction:
connection = connections[options.get('database',
DEFAULT_DB_ALIAS)]
output = '%s\n%s\n%s' % (
self.style.SQL_KEYWORD(
connection.ops.start_transaction_sql()),
output,
self.style.SQL_KEYWORD(
connection.ops.end_transaction_sql()),
)
self.stdout.write(output)
return output
def handle(self, *args, **options):
_stdout_backup, _stderr_backup = self.stdout, self.stderr
self.stdout = OutputWrapper(ConsoleOutputPlugin._stdout,
ending=CLEAR_EOL + '\n')
self.stderr = OutputWrapper(ConsoleOutputPlugin._stderr,
ending=CLEAR_EOL + '\n')
self.stderr.style_func = lambda x: Fore.LIGHTRED_EX + Back.RED + '!' + Style.RESET_ALL + ' ' + x
with bonobo.parse_args(options) as options:
services = self.get_services()
graph_coll = self.get_graph(*args, **options)
if not isinstance(graph_coll, GeneratorType):
graph_coll = (graph_coll, )
for i, graph in enumerate(graph_coll):
assert isinstance(graph,
bonobo.Graph), 'Invalid graph provided.'
print(term.lightwhite('{}. {}'.format(i + 1, graph.name)))
result = bonobo.run(graph, services=services)
print(term.lightblack(' ... return value: ' + str(result)))
print()
self.stdout, self.stderr = _stdout_backup, _stderr_backup
def validate_models(self):
import django
try:
django_setup = django.setup
except AttributeError:
pass
else:
django_setup()
s = io.StringIO()
try:
from django.core.management.validation import get_validation_errors
except ImportError:
from django.core.management.base import BaseCommand
cmd = BaseCommand()
try:
# since django 1.5
from django.core.management.base import OutputWrapper
cmd.stdout = OutputWrapper(sys.stdout)
cmd.stderr = OutputWrapper(sys.stderr)
except ImportError:
cmd.stdout, cmd.stderr = sys.stdout, sys.stderr
cmd.check()
else:
num_errors = get_validation_errors(s, None)
if num_errors:
raise RuntimeError(
'One or more Django models did not validate:\n{0}'.format(
s.getvalue()))
def __init__(
self,
fields_provider,
providers,
extra_values=None,
stdout=None,
stderr=None,
no_color=False,
):
self.fields_provider = fields_provider or PythonConfigFieldsProvider(None)
extra_values = extra_values or {}
self.extra_values = extra_values
self.providers = providers or []
self.__formatter = string.Formatter()
self.settings = {}
self.config_values = (
[]
) # list of (ConfigValue, provider_name, setting_name, final_value)
self.raw_settings = OrderedDict()
for key, value in extra_values.items():
self.raw_settings[key] = OrderedDict()
self.raw_settings[key][None] = value
# raw_settings[setting_name][str(provider) or None] = raw_value
self.__working_stack = set()
self.stdout = OutputWrapper(stdout or sys.stdout)
self.stderr = OutputWrapper(stderr or sys.stderr)
if no_color:
self.style = no_style()
else:
self.style = color_style()
self.stderr.style_func = self.style.ERROR
def server(bind='127.0.0.1', port=8000, migrate_cmd=False): # pragma: no cover
from django.utils import autoreload
if os.environ.get("RUN_MAIN") != "true":
_create_db(migrate_cmd)
User = get_user_model()
if not User.objects.filter(is_superuser=True).exists():
usr = create_user('admin',
'*****@*****.**',
'admin',
is_staff=True,
is_superuser=True)
print('')
print('A admin user (username: %s, password: admin)'
'has been created.' % usr.get_username())
print('')
from django.contrib.staticfiles.management.commands import runserver
rs = runserver.Command()
try:
from django.core.management.base import OutputWrapper
rs.stdout = OutputWrapper(sys.stdout)
rs.stderr = OutputWrapper(sys.stderr)
except ImportError:
rs.stdout = sys.stdout
rs.stderr = sys.stderr
rs.use_ipv6 = False
rs._raw_ipv6 = False
rs.addr = bind
rs.port = port
autoreload.main(
rs.inner_run, (), {
'addrport': '%s:%s' % (bind, port),
'insecure_serving': True,
'use_threading': True
})
def handle(self, *args, **options):
user = User.objects.filter(username='******').get()
org = self.create_organization(user)
sources = self.create_sources(org, user)
importer_confs = [{
'source': sources['Classes'],
'file': "./concept_classes.json"
}, {
'source': sources['Locales'],
'file': "./locales.json"
}, {
'source': sources['Datatypes'],
'file': "./datatypes_fixed.json"
}, {
'source': sources['NameTypes'],
'file': "./nametypes_fixed.json"
}, {
'source': sources['DescriptionTypes'],
'file': "./description_types.json"
}]
for conf in importer_confs:
file = open(conf['file'], 'rb')
source = conf['source']
importer = ConceptsImporter(source, file, user,
OutputWrapper(sys.stdout),
OutputWrapper(sys.stderr))
importer.import_concepts(**options)
def initialize_logging(self):
if not self.logging_initialized:
try:
self.stdout = OutputWrapper(self.stdout._out, ending='')
except AttributeError:
self.stdout = OutputWrapper(sys.stdout, ending='')
# self.stdout = codecs.getwriter('utf8')(self.stdout)
self.logging_initialized = True
def __init__(self, stdout=None, stderr=None, no_color=False):
self.stdout = OutputWrapper(stdout or sys.stdout)
self.stderr = OutputWrapper(stderr or sys.stderr)
if no_color:
self.style = no_style()
else:
self.style = color_style()
self.stderr.style_func = self.style.ERROR
def run_on_schema(schema_name,
executor_codename,
command,
function_name=None,
args=[],
kwargs={},
pass_schema_in_kwargs=False):
if not isinstance(command, BaseCommand):
# Parallel executor needs to pass command 'type' instead of 'instance'
# Therefore, no customizations for the command can be done, nor using custom stdout, stderr
command = command()
command.stdout = kwargs.pop("stdout", command.stdout)
if not isinstance(command.stdout, OutputWrapper):
command.stdout = OutputWrapper(command.stdout)
command.stderr = kwargs.pop("stderr", command.stderr)
if not isinstance(command.stderr, OutputWrapper):
command.stderr = OutputWrapper(command.stderr)
# Since we are prepending every output with the schema_name and executor, we need to determine
# whether we need to do so based on the last ending used to write. If the last write didn't end
# in '\n' then we don't do the prefixing in order to keep the output looking good.
class StyleFunc(object):
last_message = None
def __call__(self, message):
last_message = self.last_message
self.last_message = message
if last_message is None or last_message.endswith("\n"):
return "[%s:%s] %s" % (
command.style.NOTICE(executor_codename),
command.style.NOTICE(schema_name),
message,
)
return message
command.stdout.style_func = StyleFunc()
command.stderr.style_func = StyleFunc()
connections.close_all()
connection.set_schema_to(schema_name)
if pass_schema_in_kwargs:
kwargs.update({"schema_name": schema_name})
if function_name == "special:call_command":
call_command(command, *args, **kwargs)
elif function_name == "special:run_from_argv":
command.run_from_argv(args)
else:
getattr(command, function_name)(*args, **kwargs)
transaction.commit()
connection.close()
return schema_name
def __init__(self, stdout=None, stderr=None, no_color=False):
self.logging = Logging.objects.create(
name=self.name,
class_name=self.__class__,
status='Progress'
)
stdout = OutputWrapper(stdout or OutputWrapperWithLoggingCapture('stdout', self.logging))
stderr = OutputWrapper(stderr or OutputWrapperWithLoggingCapture('stderr', self.logging))
super(CommandwithLogging, self).__init__(stdout=stdout, stderr=stderr, no_color=False)
def run_migrations(args, options, executor_codename, schema_name, tenant_type='',
allow_atomic=True, idx=None, count=None):
from django.core.management import color
from django.core.management.base import OutputWrapper
from django.db import connections
style = color.color_style()
def style_func(msg):
percent_str = ''
if idx is not None and count is not None and count > 0:
percent_str = '%d/%d (%s%%) ' % (idx + 1, count, int(100 * (idx + 1) / count))
message = '[%s%s:%s] %s' % (
percent_str,
style.NOTICE(executor_codename),
style.NOTICE(schema_name),
msg
)
signal_message = '[%s%s:%s] %s' % (
percent_str,
executor_codename,
schema_name,
msg
)
schema_migrate_message.send(run_migrations, message=signal_message)
return message
connection = connections[options.get('database', get_tenant_database_alias())]
connection.set_schema(schema_name, tenant_type=tenant_type)
# ensure that django_migrations table is created in the schema before migrations run, otherwise the migration
# table in the public schema gets picked and no migrations are applied
migration_recorder = MigrationRecorder(connection)
migration_recorder.ensure_schema()
stdout = OutputWrapper(sys.stdout)
stdout.style_func = style_func
stderr = OutputWrapper(sys.stderr)
stderr.style_func = style_func
if int(options.get('verbosity', 1)) >= 1:
stdout.write(style.NOTICE("=== Starting migration"))
MigrateCommand(stdout=stdout, stderr=stderr).execute(*args, **options)
try:
transaction.commit()
connection.close()
connection.connection = None
except transaction.TransactionManagementError:
if not allow_atomic:
raise
# We are in atomic transaction, don't close connections
pass
connection.set_schema_to_public()
schema_migrated.send(run_migrations, schema_name=schema_name)
def handle(self, *args, **options):
_stdout_backup, _stderr_backup = self.stdout, self.stderr
self.stdout = OutputWrapper(ConsoleOutputPlugin._stdout, ending=CLEAR_EOL + '\n')
self.stderr = OutputWrapper(ConsoleOutputPlugin._stderr, ending=CLEAR_EOL + '\n')
self.stderr.style_func = lambda x: Fore.LIGHTRED_EX + Back.RED + '!' + Style.RESET_ALL + ' ' + x
self.run(*args, **options)
self.stdout, self.stderr = _stdout_backup, _stderr_backup
def server(bind='127.0.0.1',
port=8000,
migrate_cmd=False,
verbose=1): # pragma: no cover
try:
from channels.log import setup_logger
from channels.management.commands import runserver
logger = setup_logger('django.channels', 1)
use_channels = True
except ImportError:
from django.contrib.staticfiles.management.commands import runserver
use_channels = False
logger = None
if os.environ.get('RUN_MAIN') != 'true':
_create_db(migrate_cmd)
User = get_user_model()
if not User.objects.filter(is_superuser=True).exists():
usr = create_user('admin',
'*****@*****.**',
'admin',
is_staff=True,
is_superuser=True)
print('')
print('A admin user (username: %s, password: admin) '
'has been created.' % usr.get_username())
print('')
rs = runserver.Command()
try:
from django.core.management.base import OutputWrapper
rs.stdout = OutputWrapper(sys.stdout)
rs.stderr = OutputWrapper(sys.stderr)
except ImportError:
rs.stdout = sys.stdout
rs.stderr = sys.stderr
rs.use_ipv6 = False
rs._raw_ipv6 = False
rs.addr = bind
rs.port = port
if logger:
rs.logger = logger
if use_channels:
rs.http_timeout = 60
rs.websocket_handshake_timeout = 5
autoreload.main(
rs.inner_run, (), {
'addrport': '%s:%s' % (bind, port),
'insecure_serving': True,
'use_static_handler': True,
'use_threading': True,
'verbosity': verbose,
'use_reloader': True
})
def execute(self, *args, **options):
"""
Overridden to return response object rather than writing to stdout
"""
self.stdout = OutputWrapper(options.get('stdout', sys.stdout))
if options.get('no_color'):
self.style = no_style()
self.stderr = OutputWrapper(options.get('stderr', sys.stderr))
os.environ[str("DJANGO_COLORS")] = str("nocolor")
else:
self.stderr = OutputWrapper(options.get('stderr', sys.stderr),
self.style.ERROR)
return self.handle(*args, **options)
def handle(self, *args, **options):
_stdout_backup, _stderr_backup = self.stdout, self.stderr
self.stdout = OutputWrapper(ConsoleOutputPlugin._stdout,
ending=CLEAR_EOL + "\n")
self.stderr = OutputWrapper(ConsoleOutputPlugin._stderr,
ending=CLEAR_EOL + "\n")
self.stderr.style_func = lambda x: Fore.LIGHTRED_EX + Back.RED + "!" + Style.RESET_ALL + " " + x
try:
return self.run(*args, **options)
finally:
self.stdout, self.stderr = _stdout_backup, _stderr_backup
def handle(self, *args, **options):
if options[self.subcommand_dest] in self.instances:
command = self.instances[options[self.subcommand_dest]]
if options.get('no_color'):
command.style = no_style()
command.stderr.style_func = None
if options.get('stdout'):
command.stdout = OutputWrapper(options['stdout'])
if options.get('stderr'):
command.stderr = OutputWrapper(options.get('stderr'), command.stderr.style_func)
command.handle(*args, **options)
else:
self.print_help('manage.py', 'cms')
def handle_head(self, **options):
action = options["action"]
verbosity = options["verbosity"]
filename = options["filename"]
if filename:
self.stdout = OutputWrapper(open(filename, "w"))
self.style = no_style()
if action == "python":
self.show_python_config(verbosity)
elif action == "ini":
self.show_ini_config(verbosity)
elif action == "signals":
self.show_signals_config()
elif action == "heroku":
self.show_heroku_config()
elif action == "apache":
self.show_external_config("djangofloor/config/apache.conf")
elif action == "nginx":
self.show_external_config("djangofloor/config/nginx.conf")
elif action == "systemd":
self.show_external_config("djangofloor/config/systemd.conf")
elif action == "supervisor":
self.show_external_config("djangofloor/config/supervisor.conf")
elif action == "social_authentications":
self.show_social_auth_config()
def handle(self, *args, **options):
haystack.signal_processor = haystack.signals.BaseSignalProcessor
user = User.objects.filter(username='******').get()
org = self.create_organization(user)
sources = self.create_sources(org, user)
importer_confs = [
{'source': sources['Classes'], 'file': "./concept_classes.json"},
{'source': sources['Locales'], 'file': "./locales.json"},
{'source': sources['Datatypes'], 'file': "./datatypes_fixed.json"},
{'source': sources['NameTypes'], 'file': "./nametypes_fixed.json"},
{'source': sources['DescriptionTypes'], 'file': "./description_types.json"},
{'source': sources['MapTypes'], 'file': "./maptypes_fixed.json"}
]
update_index_required = False
for conf in importer_confs:
file = open(conf['file'], 'rb')
source = conf['source']
importer = ConceptsImporter(source, file, user, OutputWrapper(sys.stdout), OutputWrapper(sys.stderr), save_validation_errors=False)
importer.import_concepts(**options)
actions = importer.action_count
update_index_required |= actions.get(ImportActionHelper.IMPORT_ACTION_ADD, 0) > 0
update_index_required |= actions.get(ImportActionHelper.IMPORT_ACTION_UPDATE, 0) > 0
if update_index_required:
update_index.Command().handle(age=1, workers=4)
def run_from_argv(self, argv):
from google.appengine.ext.remote_api import remote_api_stub
from google.appengine.tools import appengine_rpc
import getpass
from djangae.boot import find_project_root
self.stdout = OutputWrapper(sys.stdout)
def auth_func():
return (raw_input('Google Account Login:'******'Password:'******'app.yaml')).read()
app_id = app_yaml.split("application:")[1].lstrip().split()[0]
self.stdout.write(
"Opening Remote API connection to {0}...\n".format(app_id))
remote_api_stub.ConfigureRemoteApi(
None,
'/_ah/remote_api',
auth_func,
servername='{0}.appspot.com'.format(app_id),
secure=True,
)
self.stdout.write(
"...Connection established...have a nice day :)\n".format(app_id))
argv = argv[:1] + argv[2:]
execute_from_command_line(argv)
def handle_head(self, **options):
action = options["action"]
verbosity = options["verbosity"]
filename = options["filename"]
fd = None
if filename:
fd = io.StringIO()
self.stdout = OutputWrapper(fd)
self.style = no_style()
if action == "python":
self.show_python_config(verbosity)
elif action == "ini":
self.show_ini_config(verbosity)
elif action == "env":
self.show_env_config(verbosity)
if filename and action in {"python", "env"}:
content = fd.getvalue()
# noinspection PyBroadException
if action == "python":
try:
# noinspection PyPackageRequirements,PyUnresolvedReferences
import black
mode = black.FileMode()
# noinspection PyArgumentList
content = black.format_file_contents(content,
fast=False,
mode=mode)
except Exception:
pass
with open(filename, "w") as dst_fd:
dst_fd.write(content)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Country names mapping
self.country_names = dict(Country.objects.values_list('name', 'pk'))
# Partner entity id mapping
self.entity_ids = dict(
Partner.objects.annotate(entity_id=Subquery(
EntityVersion.objects.filter(
entity__organization=OuterRef('organization_id'),
parent__isnull=True,
).order_by('-start_date').values('entity_id')[:1]),
).values_list('pk', 'entity_id'))
# Existing partner addresses
self.partners = self.get_existing_partner_addresses()
# Geocoding url
self.url = "{esb_api}/{endpoint}".format(
esb_api=settings.ESB_API_URL,
endpoint=settings.ESB_GEOCODING_ENDPOINT,
)
self.counts = defaultdict(int)
self.delayed_io = io.StringIO()
self.stdout_wrapper = OutputWrapper(self.delayed_io)
class MakemessagesCommand(DjangoMakemessagesCommand):
stdout = OutputWrapper(sys.stdout)
verbosity = 1
symlinks = False
ignore_patterns = ['CVS', '.*', '*~', '*.pyc']
no_obsolete = False
keep_pot = False
invoked_for_django = False
locale_paths = []
default_locale_path = None
def _update_locale_paths(self, app_name):
self.locale_paths = [
os.path.join(app_name, 'locale').replace('\\', '/')
]
self.default_locale_path = self.locale_paths[0]
if not os.path.exists(self.default_locale_path):
os.makedirs(self.default_locale_path)
@classmethod
def get_command(cls, app_name, domain):
assert domain in ('django', 'djangojs')
check_programs('xgettext', 'msguniq', 'msgmerge', 'msgattrib')
co = cls()
co.domain = domain
co.extensions = handle_extensions(GETTEXT_EXTENSIONS[domain])
co._update_locale_paths(app_name)
return co
def run_migrations(args,
options,
executor_codename,
schema_name,
allow_atomic=True,
idx=None,
count=None):
from django.core.management import color
from django.core.management.base import OutputWrapper
from django.db import connections
style = color.color_style()
def style_func(msg):
percent_str = ''
if idx is not None and count is not None and count > 0:
percent_str = '%d/%d (%s%%) ' % (idx + 1, count,
int(100 * (idx + 1) / count))
return '[%s%s:%s] %s' % (percent_str, style.NOTICE(executor_codename),
style.NOTICE(schema_name), msg)
include_public = True if (options.get('shared')
or schema_name == 'public') else False
connection = connections[get_tenant_database_alias()]
connection.set_schema(schema_name, include_public=include_public)
stdout = OutputWrapper(sys.stdout)
stdout.style_func = style_func
stderr = OutputWrapper(sys.stderr)
stderr.style_func = style_func
if int(options.get('verbosity', 1)) >= 1:
stdout.write(style.NOTICE("=== Starting migration"))
MigrateCommand(stdout=stdout, stderr=stderr).execute(*args, **options)
try:
transaction.commit()
connection.close()
connection.connection = None
except transaction.TransactionManagementError:
if not allow_atomic:
raise
# We are in atomic transaction, don't close connections
pass
connection.set_schema_to_public()
def multiprocess_reader(urls, stdout=None):
stdout = OutputWrapper(stdout or sys.stdout)
result = URLReader(urls=urls)()
out = set()
for built_result in result:
out.add(built_result)
stdout.write("Read {}".format(built_result.url))
return out
def handle(self, *args, **options):
_stdout_backup, _stderr_backup = self.stdout, self.stderr
self.stdout = OutputWrapper(ConsoleOutputPlugin._stdout,
ending=CLEAR_EOL + '\n')
self.stderr = OutputWrapper(ConsoleOutputPlugin._stderr,
ending=CLEAR_EOL + '\n')
self.stderr.style_func = lambda x: Fore.LIGHTRED_EX + Back.RED + '!' + Style.RESET_ALL + ' ' + x
with bonobo.parse_args(options) as options:
result = bonobo.run(
self.get_graph(*args, **options),
services=self.get_services(),
)
self.stdout, self.stderr = _stdout_backup, _stderr_backup
return '\nReturn Value: ' + str(result)
def __init__(self,
defaults=None,
specified_apps=None,
dry_run=None,
prompt_output=None):
super().__init__(defaults=defaults,
specified_apps=specified_apps,
dry_run=dry_run)
self.prompt_output = prompt_output or OutputWrapper(sys.stdout)
def write(msg, is_error=False):
stdout = OutputWrapper(sys.stdout)
style = color_style()
if is_error:
styling_msg = style.ERROR(msg)
else:
styling_msg = style.SUCCESS(msg)
stdout.write(styling_msg)
def on_execute_command(self, tenant, args, options):
style = color.color_style()
stdout = OutputWrapper(sys.stdout)
stdout.write(style.MIGRATE_HEADING("=".ljust(70, "=")))
stdout.write(
style.MIGRATE_HEADING("=== Starting collectstatic: {0} ".format(
tenant.schema_name).ljust(70, "=")))
stdout.write(style.MIGRATE_HEADING("=".ljust(70, "=")))
options["interactive"] = False
super(Command, self).on_execute_command(tenant, args, options)
