class Command(BaseCommand):
def __init__(self, *args, **kwargs):
from djangae.boot import setup_paths, setup_datastore_stubs
setup_paths()
super(Command, self).__init__(*args, **kwargs)
def run_from_argv(self, argv):
from google.appengine.ext.remote_api import remote_api_stub
from google.appengine.tools import appengine_rpc
import getpass
from djangae.boot import find_project_root
self.stdout = OutputWrapper(sys.stdout)
def auth_func():
return (raw_input('Google Account Login:'******'Password:'******'app.yaml')).read()
app_id = app_yaml.split("application:")[1].lstrip().split()[0]
self.stdout.write("Opening Remote API connection to {0}...\n".format(app_id))
remote_api_stub.ConfigureRemoteApi(None,
'/_ah/remote_api',
auth_func,
servername='{0}.appspot.com'.format(app_id),
secure=True,
)
self.stdout.write("...Connection established...have a nice day :)\n".format(app_id))
argv = argv[:1] + argv[2:]
execute_from_command_line(argv)
def multiprocess_reader(urls, stdout=None):
stdout = OutputWrapper(stdout or sys.stdout)
result = URLReader(urls=urls)()
out = set()
for built_result in result:
out.add(built_result)
stdout.write("Read {}".format(built_result.url))
return out
def initialize_logging(self):
if not self.logging_initialized:
try:
self.stdout = OutputWrapper(self.stdout._out, ending='')
except AttributeError:
self.stdout = OutputWrapper(sys.stdout, ending='')
# self.stdout = codecs.getwriter('utf8')(self.stdout)
self.logging_initialized = True
def execute(self, *args, **options):
try:
super(Command, self).execute(*args, **options)
except CommandError as e:
if options.get('traceback', False):
raise
# self.stderr is not guaranteed to be set here
stderr = getattr(self, 'stderr', None)
if not stderr:
stderr = OutputWrapper(sys.stderr, self.style.ERROR)
stderr.write('%s: %s' % (e.__class__.__name__, e))
sys.exit(2)
def initialize_logging(self):
if not self.logging_initialized:
self.stdout = OutputWrapper(
getattr(self, 'stdout', sys.stdout),
ending='')
self.stdout = codecs.getwriter('utf8')(self.stdout)
self.logging_initialized = True
def handle_head(self, **options):
action = options["action"]
verbosity = options["verbosity"]
filename = options["filename"]
if filename:
self.stdout = OutputWrapper(open(filename, "w"))
self.style = no_style()
if action == "python":
self.show_python_config(verbosity)
elif action == "ini":
self.show_ini_config(verbosity)
elif action == "signals":
self.show_signals_config()
elif action == "heroku":
self.show_heroku_config()
elif action == "apache":
self.show_external_config("djangofloor/config/apache.conf")
elif action == "nginx":
self.show_external_config("djangofloor/config/nginx.conf")
elif action == "systemd":
self.show_external_config("djangofloor/config/systemd.conf")
elif action == "supervisor":
self.show_external_config("djangofloor/config/supervisor.conf")
elif action == "social_authentications":
self.show_social_auth_config()
def handle(self, *args, **options):
haystack.signal_processor = haystack.signals.BaseSignalProcessor
user = User.objects.filter(username='******').get()
org = self.create_organization(user)
sources = self.create_sources(org, user)
importer_confs = [
{'source': sources['Classes'], 'file': "./concept_classes.json"},
{'source': sources['Locales'], 'file': "./locales.json"},
{'source': sources['Datatypes'], 'file': "./datatypes_fixed.json"},
{'source': sources['NameTypes'], 'file': "./nametypes_fixed.json"},
{'source': sources['DescriptionTypes'], 'file': "./description_types.json"},
{'source': sources['MapTypes'], 'file': "./maptypes_fixed.json"}
]
update_index_required = False
for conf in importer_confs:
file = open(conf['file'], 'rb')
source = conf['source']
importer = ConceptsImporter(source, file, user, OutputWrapper(sys.stdout), OutputWrapper(sys.stderr), save_validation_errors=False)
importer.import_concepts(**options)
actions = importer.action_count
update_index_required |= actions.get(ImportActionHelper.IMPORT_ACTION_ADD, 0) > 0
update_index_required |= actions.get(ImportActionHelper.IMPORT_ACTION_UPDATE, 0) > 0
if update_index_required:
update_index.Command().handle(age=1, workers=4)
def execute(self, *args, **options):
try:
super(Command, self).execute(*args, **options)
except CommandError as e:
if options.get('traceback', False):
raise
# self.stderr is not guaranteed to be set here
stderr = getattr(self, 'stderr', None)
if not stderr:
if HAS_OUTPUTWRAPPER:
stderr = OutputWrapper(sys.stderr, self.style.ERROR)
else:
stderr = sys.stderr
stderr.write('%s: %s' % (e.__class__.__name__, e))
sys.exit(2)
class QuestionerHelperMethodsTests(SimpleTestCase):
def setUp(self):
self.prompt = OutputWrapper(StringIO())
self.questioner = InteractiveMigrationQuestioner(prompt_output=self.prompt)
@mock.patch("builtins.input", return_value="datetime.timedelta(days=1)")
def test_questioner_default_timedelta(self, mock_input):
value = self.questioner._ask_default()
self.assertEqual(value, datetime.timedelta(days=1))
@mock.patch("builtins.input", return_value="")
def test_questioner_default_no_user_entry(self, mock_input):
value = self.questioner._ask_default(default="datetime.timedelta(days=1)")
self.assertEqual(value, datetime.timedelta(days=1))
@mock.patch("builtins.input", side_effect=["", "exit"])
def test_questioner_no_default_no_user_entry(self, mock_input):
with self.assertRaises(SystemExit):
self.questioner._ask_default()
self.assertIn(
"Please enter some code, or 'exit' (without quotes) to exit.",
self.prompt.getvalue(),
)
@mock.patch("builtins.input", side_effect=["bad code", "exit"])
def test_questioner_no_default_bad_user_entry_code(self, mock_input):
with self.assertRaises(SystemExit):
self.questioner._ask_default()
self.assertIn("Invalid input: ", self.prompt.getvalue())
@mock.patch("builtins.input", side_effect=["", "n"])
def test_questioner_no_default_no_user_entry_boolean(self, mock_input):
value = self.questioner._boolean_input("Proceed?")
self.assertIs(value, False)
@mock.patch("builtins.input", return_value="")
def test_questioner_default_no_user_entry_boolean(self, mock_input):
value = self.questioner._boolean_input("Proceed?", default=True)
self.assertIs(value, True)
@mock.patch("builtins.input", side_effect=[10, "garbage", 1])
def test_questioner_bad_user_choice(self, mock_input):
question = "Make a choice:"
value = self.questioner._choice_input(question, choices="abc")
expected_msg = f"{question}\n" f" 1) a\n" f" 2) b\n" f" 3) c\n"
self.assertIn(expected_msg, self.prompt.getvalue())
self.assertEqual(value, 1)
def run_migrations(args, options, executor_codename, schema_name, tenant_type='',
allow_atomic=True, idx=None, count=None):
from django.core.management import color
from django.core.management.base import OutputWrapper
from django.db import connections
style = color.color_style()
def style_func(msg):
percent_str = ''
if idx is not None and count is not None and count > 0:
percent_str = '%d/%d (%s%%) ' % (idx + 1, count, int(100 * (idx + 1) / count))
message = '[%s%s:%s] %s' % (
percent_str,
style.NOTICE(executor_codename),
style.NOTICE(schema_name),
msg
)
signal_message = '[%s%s:%s] %s' % (
percent_str,
executor_codename,
schema_name,
msg
)
schema_migrate_message.send(run_migrations, message=signal_message)
return message
connection = connections[options.get('database', get_tenant_database_alias())]
connection.set_schema(schema_name, tenant_type=tenant_type)
# ensure that django_migrations table is created in the schema before migrations run, otherwise the migration
# table in the public schema gets picked and no migrations are applied
migration_recorder = MigrationRecorder(connection)
migration_recorder.ensure_schema()
stdout = OutputWrapper(sys.stdout)
stdout.style_func = style_func
stderr = OutputWrapper(sys.stderr)
stderr.style_func = style_func
if int(options.get('verbosity', 1)) >= 1:
stdout.write(style.NOTICE("=== Starting migration"))
MigrateCommand(stdout=stdout, stderr=stderr).execute(*args, **options)
try:
transaction.commit()
connection.close()
connection.connection = None
except transaction.TransactionManagementError:
if not allow_atomic:
raise
# We are in atomic transaction, don't close connections
pass
connection.set_schema_to_public()
schema_migrated.send(run_migrations, schema_name=schema_name)
def on_execute_command(self, tenant, args, options):
style = color.color_style()
stdout = OutputWrapper(sys.stdout)
stdout.write(style.MIGRATE_HEADING("=".ljust(70, "=")))
stdout.write(
style.MIGRATE_HEADING("=== Starting collectstatic: {0} ".format(
tenant.schema_name).ljust(70, "=")))
stdout.write(style.MIGRATE_HEADING("=".ljust(70, "=")))
options["interactive"] = False
super(Command, self).on_execute_command(tenant, args, options)
def handle(self, *args, **options):
_stdout_backup, _stderr_backup = self.stdout, self.stderr
self.stdout = OutputWrapper(ConsoleOutputPlugin._stdout,
ending=CLEAR_EOL + '\n')
self.stderr = OutputWrapper(ConsoleOutputPlugin._stderr,
ending=CLEAR_EOL + '\n')
self.stderr.style_func = lambda x: Fore.LIGHTRED_EX + Back.RED + '!' + Style.RESET_ALL + ' ' + x
with bonobo.parse_args(options) as options:
result = bonobo.run(
self.get_graph(*args, **options),
services=self.get_services(),
)
self.stdout, self.stderr = _stdout_backup, _stderr_backup
return '\nReturn Value: ' + str(result)
def __init__(self,
defaults=None,
specified_apps=None,
dry_run=None,
prompt_output=None):
super().__init__(defaults=defaults,
specified_apps=specified_apps,
dry_run=dry_run)
self.prompt_output = prompt_output or OutputWrapper(sys.stdout)
def on_execute_command(self, tenant, args, options):
style = color.color_style()
stdout = OutputWrapper(sys.stdout)
stdout.write(style.MIGRATE_HEADING("=".ljust(70, "=")))
stdout.write(style.MIGRATE_HEADING("=== Starting collectstatic: {0} ".format(tenant.schema_name).ljust(70, "=")))
stdout.write(style.MIGRATE_HEADING("=".ljust(70, "=")))
options["interactive"] = False
super(Command, self).on_execute_command(tenant, args, options)
def __init__(self, options):
self.stdout = OutputWrapper(options.get('stdout', sys.stdout))
self.keyword_based_reporting_orgs = {
'WASH Alliance': 8,
'Connect4Change': 34,
'SRHR Alliance': 1043,
'WvW': 43,
'wvw2014': 43,
'wvw2015': 43,
'wvw2016': 43,
'WfW': 43,
'wfw2014': 43,
'wfw2015': 43,
'wfw2016': 43,
}
self.keywords_set = set(self.keyword_based_reporting_orgs.keys())
self.migrate = options['migrate']
self.ok_list = []
self.fix_list = []
def main():
parser = OptionParser(
usage=textwrap.dedent("""\
%prog [options] <instance-name> [<target-directory>]
Creates a new EOxServer instance with all necessary files and
folder structure. Optionally, a SQLite database is initiated.
"""),
version=eoxserver.get_version()
)
parser.add_option('-i', '--init-spatialite', '--init_spatialite',
dest='init_spatialite', action='store_true', default=False,
help='Flag to initialize the sqlite database.'
)
parser.add_option('-v', '--verbosity',
action='store', dest='verbosity', default='1',
type='choice', choices=['0', '1', '2', '3']
)
parser.add_option('--traceback',
action='store_true', help='Raise on exception'
)
options, args = parser.parse_args()
error_stream = OutputWrapper(sys.stderr, color_style().ERROR)
if not args:
error_stream.write("Mandatory argument 'instance-name' not given.\n")
sys.exit(1)
name = args[0]
try:
target = args[1]
except IndexError:
target = None
try:
create_instance(name, target, **options.__dict__)
except Exception as e:
if options.traceback:
raise
error_stream.write("%s: %s\n" % (e.__class__.__name__, e))
def __init__(
self, fields_provider, providers, stdout=None, stderr=None, no_color=False,
):
self.fields_provider = fields_provider or PythonConfigFieldsProvider(None)
self.providers = providers or []
self.__formatter = string.Formatter()
self.settings = {}
self.config_values = (
[]
) # list of (ConfigValue, provider_name, setting_name, final_value)
self.raw_settings = OrderedDict()
# raw_settings[setting_name][str(provider) or None] = raw_value
self.__working_stack = set()
self.stdout = OutputWrapper(stdout or sys.stdout)
self.stderr = OutputWrapper(stderr or sys.stderr)
if no_color:
self.style = no_style()
else:
self.style = color_style()
self.stderr.style_func = self.style.ERROR
def handle(self, *args, **options):
user = User.objects.filter(username='******').get()
org = self.create_organization(user, 'CIEL')
source = self.create_source(user, org, 'CIEL')
demo_file = open('./demo-data/ciel_20180601_concepts_2k.json', 'rb')
importer = ConceptsImporter(source,
demo_file,
user,
OutputWrapper(sys.stdout),
OutputWrapper(sys.stderr),
save_validation_errors=False)
importer.import_concepts(**options)
demo_file = open('./demo-data/ciel_20180601_mappings_2k.json', 'rb')
importer = MappingsImporter(source, demo_file,
OutputWrapper(sys.stdout),
OutputWrapper(sys.stderr), user)
importer.import_mappings(**options)
def run_migrations(args, options, executor_codename, schema_name, allow_atomic=True):
from django.core.management import color
from django.core.management.base import OutputWrapper
from django.db import connection
style = color.color_style()
def style_func(msg):
return '[%s:%s] %s' % (
style.NOTICE(executor_codename),
style.NOTICE(schema_name),
msg
)
connection.set_schema(schema_name)
stdout = OutputWrapper(sys.stdout)
stdout.style_func = style_func
stderr = OutputWrapper(sys.stderr)
stderr.style_func = style_func
if int(options.get('verbosity', 1)) >= 1:
stdout.write(style.NOTICE("=== Starting migration"))
MigrateCommand(stdout=stdout, stderr=stderr).execute(*args, **options)
try:
transaction.commit()
connection.close()
connection.connection = None
except transaction.TransactionManagementError:
if not allow_atomic:
raise
# We are in atomic transaction, don't close connections
pass
connection.set_schema_to_public()
def execute(self, *args, **options):
"""
Try to execute this command, performing model validation if
needed (as controlled by the attribute
``self.requires_model_validation``, except if force-skipped).
"""
# Switch to English, because django-admin.py creates database content
# like permissions, and those shouldn't contain any translations.
# But only do this if we can assume we have a working settings file,
# because django.utils.translation requires settings.
saved_lang = None
self.stdout = OutputWrapper(options.get('stdout', sys.stdout))
self.stderr = OutputWrapper(options.get('stderr', sys.stderr), self.style.ERROR)
if self.can_import_settings:
from django.utils import translation
saved_lang = translation.get_language()
translation.activate('en-us')
try:
if getattr(self, "requires_model_validation", False) and not options.get('skip_validation'):
self.validate()
result = self.handle(*args, **options)
if isinstance(result, basestring):
if self.output_transaction:
# This needs to be imported here, because it relies on
# settings.
from django.db import connections, DEFAULT_DB_ALIAS
connection = connections[options.get('database', DEFAULT_DB_ALIAS)]
if connection.ops.start_transaction_sql():
self.stdout.write(self.style.SQL_KEYWORD(connection.ops.start_transaction_sql()))
self.stdout.write(result)
if self.output_transaction:
self.stdout.write('\n' + self.style.SQL_KEYWORD("COMMIT;"))
result = 0
return result
finally:
if saved_lang is not None:
translation.activate(saved_lang)
class ImportData:
def __init__(self):
self.stdout = OutputWrapper(sys.stdout)
self.stderr = OutputWrapper(sys.stderr)
def init_sampledataset(self, target_lists):
"""
:param target_lists:
:return:
"""
self.stdout.write('Loading data...')
for file in target_lists:
self.stdout.write(f'file: {file}')
with open(file) as f:
reader = csv.reader(f)
# skip header
next(reader, None)
for row in reader:
data = {
'date': datetime.strptime(row[0], '%d.%m.%Y').date(),
'channel': row[1],
'country': row[2],
'os': row[3],
'impressions': row[4],
'clicks': row[5],
'installs': row[6],
'spend': Decimal(str(row[7])),
'revenue': Decimal(str(row[8])),
}
SampleDataset.objects.update_or_create(**data)
self.stdout.write('Done!')
def execute(self, *args, **options):
saved_lang = None
self.stdout = OutputWrapper(options.get('stdout', sys.stdout))
self.stderr = OutputWrapper(options.get('stderr', sys.stderr),
self.style.ERROR)
if self.can_import_settings:
from django.utils import translation
saved_lang = translation.get_language()
translation.activate('en-us')
try:
if self.requires_model_validation and not options.get(
'skip_validation'):
self.validate()
output = self.handle(*args, **options)
finally:
if saved_lang is not None:
translation.activate(saved_lang)
return output
def handle(self, *args, **options):
user = User.objects.filter(username='******').get()
org = self.create_organization(user)
sources = self.create_sources(org, user)
importer_confs = [
{
'source': sources['Classes'],
'file': "./concept_classes.json"
},
{
'source': sources['Locales'],
'file': "./locales.json"
},
{
'source': sources['Datatypes'],
'file': "./datatypes_fixed.json"
},
{
'source': sources['NameTypes'],
'file': "./nametypes_fixed.json"
},
{
'source': sources['DescriptionTypes'],
'file': "./description_types.json"
},
#{'source': sources['MapTypes'], 'file': "./maptypes_fixed.json"}
]
for conf in importer_confs:
file = open(conf['file'], 'rb')
source = conf['source']
importer = ConceptsImporter(source, file, user,
OutputWrapper(sys.stdout),
OutputWrapper(sys.stderr))
importer.import_concepts(**options)
def handle(self, *args, **options):
user = self.create_admin_user()
version = 'v2019-07-01'
source_version = SourceVersion.objects.filter(mnemonic=version)
if source_version.exists():
return
org = self.create_organization(user, 'CIEL')
source = self.create_source(user, org, 'CIEL')
demo_file = open('./demo-data/ciel_' + version + '_c2k.json', 'rb')
importer = ConceptsImporter(source,
demo_file,
user,
OutputWrapper(sys.stdout),
OutputWrapper(sys.stderr),
save_validation_errors=False)
importer.import_concepts(**options)
demo_file = open('./demo-data/ciel_' + version + '_m2k.json', 'rb')
importer = MappingsImporter(source, demo_file,
OutputWrapper(sys.stdout),
OutputWrapper(sys.stderr), user)
importer.import_mappings(**options)
new_version = SourceVersion.for_base_object(source,
version,
released=True)
SourceVersion.persist_new(new_version,
versioned_object=source,
force_insert=True)
update_children_for_resource_version.delay(new_version.id, 'source')
export_source.delay(new_version.id)
def run_from_argv(self, argv):
parser = self.create_parser(argv[0], argv[1])
options = parser.parse_args(argv[2:])
handle_default_options(options)
try:
self.execute(**options.__dict__)
except Exception as e:
if options.traceback or not isinstance(e, CommandError):
raise
# self.stderr is not guaranteed to be set here
stderr = getattr(self, 'stderr',
OutputWrapper(sys.stderr, self.style.ERROR))
stderr.write('%s: %s' % (e.__class__.__name__, e))
sys.exit(1)
def run_migrations(args,
options,
executor_codename,
schema_name,
allow_atomic=True,
idx=None,
count=None):
from django.core.management import color
from django.core.management.base import OutputWrapper
from django.db import connections
style = color.color_style()
def style_func(msg):
percent_str = ''
if idx is not None and count is not None and count > 0:
percent_str = '%d/%d (%s%%) ' % (idx + 1, count,
int(100 * (idx + 1) / count))
return '[%s%s:%s] %s' % (percent_str, style.NOTICE(executor_codename),
style.NOTICE(schema_name), msg)
include_public = True if (options.get('shared')
or schema_name == 'public') else False
connection = connections[get_tenant_database_alias()]
connection.set_schema(schema_name, include_public=include_public)
stdout = OutputWrapper(sys.stdout)
stdout.style_func = style_func
stderr = OutputWrapper(sys.stderr)
stderr.style_func = style_func
if int(options.get('verbosity', 1)) >= 1:
stdout.write(style.NOTICE("=== Starting migration"))
MigrateCommand(stdout=stdout, stderr=stderr).execute(*args, **options)
try:
transaction.commit()
connection.close()
connection.connection = None
except transaction.TransactionManagementError:
if not allow_atomic:
raise
# We are in atomic transaction, don't close connections
pass
connection.set_schema_to_public()
def run_from_argv(self, argv):
arguments = docopt(self.docs, argv[2:])
self._handle_default_options(arguments)
try:
self.execute(*[], **arguments)
except Exception as e:
# self.stderr is not guaranteed to be set here
stderr = getattr(self, 'stderr',
OutputWrapper(sys.stderr, self.style.ERROR))
if arguments.get('--traceback', False):
stderr.write(traceback.format_exc())
else:
stderr.write('%s: %s' % (e.__class__.__name__, e))
sys.exit(1)
def multiprocess_writer(data, stdout=None):
stdout = OutputWrapper(stdout or sys.stdout)
result = URLWriter(data=data)()
out = set()
for built_result in result:
out.add(built_result)
if built_result.created:
stdout.write("Created {}".format(built_result.name))
elif built_result.modified:
stdout.write("Updated {}".format(built_result.name))
return out
def run_from_argv(self, argv):
"""
Set up any environment changes requested (e.g., Python path
and Django settings), then run this command. If the
command raises a ``CommandError``, intercept it and print it sensibly
to stderr. If the ``--traceback`` option is present or the raised
``Exception`` is not ``CommandError``, raise it.
"""
options = None
try:
if argv[2:] and not argv[2].startswith("-"):
project_code = argv[2]
try:
Project.objects.get(code=project_code)
except Project.DoesNotExist:
raise CommandError("Unrecognised project: %s" %
project_code)
if argv[3:]:
subcommand = argv[3]
else:
subcommand = "info"
try:
subcommand = self.subcommands[subcommand]()
except KeyError:
raise CommandError("Unrecognised command: %s" % subcommand)
return subcommand.run_from_argv(argv[:1] +
[subcommand, project_code] +
argv[4:])
parser = self.create_parser(argv[0], argv[1])
options, args = parser.parse_args(argv[2:])
handle_default_options(options)
self.execute(*args, **options.__dict__)
except Exception as e:
do_raise = ("--traceback" in argv
or (options and options.traceback
or not isinstance(e, CommandError)))
if do_raise:
raise
# self.stderr is not guaranteed to be set here
stderr = getattr(self, 'stderr',
OutputWrapper(sys.stderr, self.style.ERROR))
stderr.write('%s: %s' % (e.__class__.__name__, e))
sys.exit(1)
def __init__(self, broker, **kwargs):
# initialize client
self.broker = broker
self.stdout = kwargs.pop('stdout', OutputWrapper(sys.stdout))
self.style = kwargs.pop('style', color_style())
self.verbosity = kwargs.pop('verbosity', 1)
self.debug = kwargs.pop('debug', False)
self.forgive_mid = False
if self.broker['CLIENT_ID']:
self.client = mqtt.Client(self.broker['CLIENT_ID'],
self.broker['CLEAN_SESSION'])
else:
self.client = mqtt.Client()
# handle will message
if 'WILL' in self.broker:
will = self.broker['WILL']
self.client.will_set(will['topic'],
payload = will.get('payload', None),
qos = will.get('qos', 2),
retain = will.get('retain', True))
self.client.on_connect = self.on_connect
self.subscribed = {}
self.published = {}
self.client.on_publish = self.on_publish
self.client.on_subscribe = self.on_subscribe
self.client.on_disconnect = self.on_disconnect
# default message handler
self.client.on_message = self.on_message
if self.broker['USERNAME'] and self.broker['PASSWORD']:
self.client.username_pw_set(self.broker['USERNAME'],
self.broker['PASSWORD'])
self.connected = False
self.client.connect(self.broker['HOST'],
self.broker['PORT'],
self.broker['KEEPALIVE'])
def run_migrations(args,
options,
executor_codename,
schema_name,
allow_atomic=True):
from django.core.management import color
from django.core.management.base import OutputWrapper
from django.db import connections
PUBLIC_SCHEMA_NAME = get_public_schema_name()
options['database'] = settings.TENANT_DATABASE
if schema_name == PUBLIC_SCHEMA_NAME:
options['database'] = DEFAULT_DB_ALIAS
style = color.color_style()
def style_func(msg):
return '[%s:%s:%s] %s' % (options['database'],
style.NOTICE(executor_codename),
style.NOTICE(schema_name), msg)
connections[options['database']].set_schema(schema_name)
stdout = OutputWrapper(sys.stdout)
stdout.style_func = style_func
stderr = OutputWrapper(sys.stderr)
stderr.style_func = style_func
if int(options.get('verbosity', 1)) >= 1:
stdout.write(style.NOTICE("=== Starting migration"))
MigrateCommand(stdout=stdout, stderr=stderr).execute(*args, **options)
try:
transaction.commit()
connections[options['database']].close()
connections[options['database']].connection = None
except transaction.TransactionManagementError:
if not allow_atomic:
raise
# We are in atomic transaction, don't close connections
pass
def __init__(self, options):
self.stdout = OutputWrapper(options.get('stdout', sys.stdout))
self.keyword_based_reporting_orgs = {
'WASH Alliance': 8,
'Connect4Change': 34,
'SRHR Alliance': 1043,
'WvW': 43,
'wvw2014': 43,
'wvw2015': 43,
'wvw2016': 43,
'WfW': 43,
'wfw2014': 43,
'wfw2015': 43,
'wfw2016': 43,
}
self.keywords_set = set(self.keyword_based_reporting_orgs.keys())
self.migrate = options['migrate']
self.ok_list = []
self.fix_list = []
def run_migrations(args,
options,
executor_codename,
schema_name,
allow_atomic=True):
from django.core.management import color
from django.core.management.base import OutputWrapper
from django.db import connection
style = color.color_style()
def style_func(msg):
return '[%s:%s] %s' % (style.NOTICE(executor_codename),
style.NOTICE(schema_name), msg)
stdout = OutputWrapper(sys.stdout)
stdout.style_func = style_func
stderr = OutputWrapper(sys.stderr)
stderr.style_func = style_func
if int(options.get('verbosity', 1)) >= 1:
stdout.write(
style.NOTICE("=== Running migrate for schema %s" % schema_name))
connection.set_schema(schema_name)
MigrateCommand(stdout=stdout, stderr=stderr).execute(*args, **options)
try:
transaction.commit()
connection.close()
connection.connection = None
except transaction.TransactionManagementError:
if not allow_atomic:
raise
# We are in atomic transaction, don't close connections
pass
connection.set_schema_to_public()
def main():
parser = OptionParser(usage=textwrap.dedent("""\
%prog [options] <instance-name> [<target-directory>]
Creates a new EOxServer instance with all necessary files and
folder structure. Optionally, a SQLite database is initiated.
"""),
version=eoxserver.get_version())
parser.add_option('-i',
'--init-spatialite',
'--init_spatialite',
dest='init_spatialite',
action='store_true',
default=False,
help='Flag to initialize the sqlite database.')
parser.add_option('-v',
'--verbosity',
action='store',
dest='verbosity',
default='1',
type='choice',
choices=['0', '1', '2', '3'])
parser.add_option('--traceback',
action='store_true',
help='Raise on exception')
options, args = parser.parse_args()
error_stream = OutputWrapper(sys.stderr, color_style().ERROR)
if not args:
error_stream.write("Mandatory argument 'instance-name' not given.\n")
sys.exit(1)
name = args[0]
try:
target = args[1]
except IndexError:
target = None
try:
create_instance(name, target, **options.__dict__)
except Exception as e:
if options.traceback:
raise
error_stream.write("%s: %s\n" % (e.__class__.__name__, e))
def run_from_argv(self, argv):
"""
Set up any environment changes requested (e.g., Python path
and Django settings), then run this command. If the
command raises a ``CommandError``, intercept it and print it sensibly
to stderr.
"""
parser = self.create_parser(argv[0], argv[1])
args = parser.parse_args(argv[2:])
handle_default_options(args)
try:
self.execute(args)
except Exception as e:
# self.stderr is not guaranteed to be set here
try:
fallback_stderr = OutputWrapper(sys.stderr, self.style.ERROR)
except:
fallback_stderr = self.stdout
stderr = getattr(self, 'stderr', fallback_stderr)
if args.traceback:
stderr.write(traceback.format_exc())
else:
stderr.write('%s: %s' % (e.__class__.__name__, e))
sys.exit(1)
class CompilemessagesCommand(DjangoCompilemessagesCommand):
stdout = OutputWrapper(sys.stdout)
verbosity = 1
@classmethod
def compilemessages(cls):
check_programs('msgfmt')
basedirs = [
os.path.join(app, 'locale').replace('\\', '/') for app in APPS
]
co = cls()
for basedir in basedirs:
dirs = [
os.path.join(basedir, locale,
'LC_MESSAGES').replace('\\', '/')
for locale in LANGUAGES
]
locations = []
for ldir in dirs:
for dirpath, dirnames, filenames in os.walk(ldir):
locations.extend(
(dirpath, f) for f in filenames if f.endswith('.po'))
if locations:
co.compile_messages(locations)
class Progress(BaseCommand, ABC):
stdout = OutputWrapper(sys.stdout)
style = color_style()
@classmethod
def show_progress(cls, progress, instance_name):
width = 40
points = int(width * progress)
backspaces = width - points
bar = ('[' + '.' * points + ' ' * backspaces + '] ' + str(
int(progress * 100)) + ' %')
text = f'Populating {instance_name} '.ljust(25)
cls.stdout.write(
cls.style.SUCCESS(text + bar),
ending='\r'
)
@classmethod
def report_success(cls, number, instance_name):
cls.stdout.write(' ' * 100, ending='\r')
cls.stdout.write(
cls.style.SUCCESS(
f'Successfully created {number} {instance_name}'
))
def __init__(self, stdout=None, stderr=None, no_color=False):
super().__init__(stdout, stderr, no_color)
# 检测目录位置
if self.NEED_PROJECT:
settings_path = os.path.join(os.getcwd(), 'deeru')
settings_py = os.path.join(settings_path, 'settings.py')
if not os.path.exists(settings_py):
raise CommandError('该命令需要在工程目录下运行')
self.error = self.stderr.write
info_out = OutputWrapper(sys.stdout)
info_out.style_func = self.style.WARNING
self.info = info_out.write
success_out = OutputWrapper(sys.stdout)
success_out.style_func = self.style.SUCCESS
self.success = success_out.write
def run_migrations(args, options, executor_codename, schema_name, allow_atomic=True, idx=None, count=None):
from django.core.management import color
from django.core.management.base import OutputWrapper
from django.db import connections
style = color.color_style()
def style_func(msg):
percent_str = ''
if idx is not None and count is not None and count > 0:
percent_str = '%d/%d (%s%%) ' % (idx + 1, count, int(100 * (idx + 1) / count))
return '[%s%s:%s] %s' % (
percent_str,
style.NOTICE(executor_codename),
style.NOTICE(schema_name),
msg
)
connection = connections[get_tenant_database_alias()]
connection.set_schema(schema_name)
stdout = OutputWrapper(sys.stdout)
stdout.style_func = style_func
stderr = OutputWrapper(sys.stderr)
stderr.style_func = style_func
if int(options.get('verbosity', 1)) >= 1:
stdout.write(style.NOTICE("=== Starting migration"))
MigrateCommand(stdout=stdout, stderr=stderr).execute(*args, **options)
try:
transaction.commit()
connection.close()
connection.connection = None
except transaction.TransactionManagementError:
if not allow_atomic:
raise
# We are in atomic transaction, don't close connections
pass
connection.set_schema_to_public()
def __init__(self, stdout=None, style=None):
self.user = None
self.devnull = open(os.devnull, "w")
self.stdout = stdout if stdout else OutputWrapper(self.devnull)
self.style = style if style else no_style()
def handle(self, *args, **options):
self.error = self.stderr.write
info_out = OutputWrapper(sys.stdout)
info_out.style_func = self.style.WARNING
self.info = info_out.write
success_out = OutputWrapper(sys.stdout)
success_out.style_func = self.style.SUCCESS
self.success = success_out.write
self.info("初始化中：")
try:
os.mkdir('log')
except:
pass
with open('./log/init.log', 'a')as f:
f.write('开始初始化(%s)\n' % str(now()))
# ============
self.info('初始化数据库 ... ')
with open('./log/init.log', 'a')as f:
f.write('初始化数据库\n')
try:
management.call_command('migrate', stdout=f)
# management.call_command('makemigrations', 'app', stdout=f)
except:
traceback.print_exc(file=f)
self.error('初始化数据库 ... [失败]，更多信息查看 ./log/init.log ')
raise
self.success('初始化数据库 ... [完成]')
# ============
self.info('初始化静态文件 ... ')
with open('./log/init.log', 'a')as f:
f.write('初始化静态文件\n')
try:
management.call_command('collectstatic', '--noinput', stdout=f)
except:
traceback.print_exc(file=f)
self.error('初始化静态文件 ... [失败]，更多信息查看 ./log/init.log ')
raise
self.success('初始化静态文件 ... [完成]')
# ============
self.info('创建管理员账户 ... ')
username = input('输入管理账户登录名（默认：admin）：')
if not username:
username = '******'
password = getpass('输入密码（默认：123456）：')
if not password:
password = '******'
with open('./log/init.log', 'a')as f:
try:
flag = True
User._default_manager.db_manager('default').create_superuser(username, None, password)
except:
flag = False
traceback.print_exc(3)
traceback.print_exc(file=f)
self.error('创建管理员账户 ... [失败]，更多信息查看 ./log/init.log ')
finally:
if not flag:
return
# ============
self.success('创建管理员账户 ... [完成]')
self.success('\n初始化完成 ！！')
def parse_logs(qs, stdout=None):
"""
Parse logs for kudos.
"""
names = collections.deque(maxlen=200)
unattributed = 0
count = 0
kudos = {}
kudos_count = 0
kudos_first = {}
kudos_recent = {}
if stdout and not isinstance(stdout, OutputWrapper):
stdout = OutputWrapper(stdout)
def set_thanked(nick):
timestamp = log[3]
kudos[nick] = kudos.get(nick, 0) + 1
kudos_first.setdefault(nick, timestamp)
kudos_recent[nick] = timestamp
qs = qs.order_by('pk').filter(command='PRIVMSG')
qs = qs.values_list('pk', 'nick', 'text', 'timestamp')
for log in _iterate_log(qs):
log_nick = log[1].lower()
log_text = log[2]
count += 1
directed = directed_message(log_text)
if directed:
directed = directed.lower()
if directed == log_nick:
# Can't thank yourself :P
directed = None
if RE_KUDOS.search(log_text):
kudos_count += 1
attributed = False
if directed:
for nick, _ in names:
if nick == directed:
set_thanked(nick)
attributed = True
break
if not attributed:
lower_text = log_text.lower()
for recent in (
bits[0] for bits in names if bits[0] != log_nick):
re_text = '(?:^| )@?{}(?:$|\W)'.format(re.escape(recent))
if re.search(re_text, lower_text):
set_thanked(recent)
attributed = True
if not attributed:
for nick, directed in names:
if directed == log_nick:
set_thanked(nick)
attributed = True
break
if not attributed:
unattributed += 1
names.append((log_nick, directed))
if stdout and not count % 10000:
stdout.write('.', ending='')
stdout.flush()
if stdout:
stdout.write('')
kudos_list = []
for c, nick in sorted((c, nick) for nick, c in kudos.items()):
kudos_list.append({
'nick': nick,
'count': c,
'first': kudos_first[nick],
'recent': kudos_recent[nick]
})
return {
'kudos': kudos_list,
'message_count': count,
'kudos_given': kudos_count,
'unattributed': unattributed,
}
class Command(BaseCommand):
help = (
"show the current configuration."
'Can display as python file ("config python") or as .ini file ("config ini"). Use -v 2 to display more info.'
)
requires_system_checks = False
options = {
"python": "display the current config as Python module",
"ini": "display the current config as .ini file",
"heroku": "display a configuration valid to deploy on Heroku",
"apache": "display an example of Apache config",
"nginx": "display an example of Nginx config",
"systemd": "display an example of systemd config",
"supervisor": "display an example of Supervisor config",
"social_authentications": "display configured social authentications",
}
if settings.USE_CELERY:
options["signals"] = "show the defined signals and remote functions"
def add_arguments(self, parser):
assert isinstance(parser, ArgumentParser)
parser.add_argument(
"action",
default="show",
choices=self.options,
help=",\n".join(['"%s": %s' % x for x in self.options.items()]),
)
parser.add_argument(
"--filename", default=None, help="write output to this file"
)
remove_arguments_from_help(
parser, {"--settings", "--traceback", "--pythonpath"}
)
def handle(self, *args, **options):
try:
self.handle_head(**options)
except BrokenPipeError:
pass
def handle_head(self, **options):
action = options["action"]
verbosity = options["verbosity"]
filename = options["filename"]
if filename:
self.stdout = OutputWrapper(open(filename, "w"))
self.style = no_style()
if action == "python":
self.show_python_config(verbosity)
elif action == "ini":
self.show_ini_config(verbosity)
elif action == "signals":
self.show_signals_config()
elif action == "heroku":
self.show_heroku_config()
elif action == "apache":
self.show_external_config("djangofloor/config/apache.conf")
elif action == "nginx":
self.show_external_config("djangofloor/config/nginx.conf")
elif action == "systemd":
self.show_external_config("djangofloor/config/systemd.conf")
elif action == "supervisor":
self.show_external_config("djangofloor/config/supervisor.conf")
elif action == "social_authentications":
self.show_social_auth_config()
def show_external_config(self, config):
content = render_to_string(config, merger.settings)
self.stdout.write(content)
def show_signals_config(self):
import_signals_and_functions()
def display_callable(conn):
fn = conn.function
if getattr(fn, "__module__", None) and getattr(fn, "__name__", None):
path = "%s.%s" % (fn.__module__, fn.__name__)
elif getattr(fn, "__name__", None):
path = fn.__name__
else:
path = str(fn)
return path
self.stdout.write(self.style.ERROR("Signals"))
data = list(decorators.REGISTERED_SIGNALS.items())
for name, connections in sorted(data, key=lambda x: x[0]):
self.stdout.write(self.style.WARNING(' "%s"' % name))
for connection in connections:
self.stdout.write(
self.style.NOTICE(" -> %s" % display_callable(connection))
)
self.stdout.write(self.style.ERROR("Functions"))
data = list(decorators.REGISTERED_FUNCTIONS.items())
for name, connection in sorted(data, key=lambda x: x[0]):
self.stdout.write(
self.style.WARNING(
' "%s" -> %s' % (name, display_callable(connection))
)
)
def show_ini_config(self, verbosity):
if verbosity >= 2:
self.stdout.write(self.style.SUCCESS("# read configuration files:"))
for provider in merger.providers:
if not isinstance(provider, IniConfigProvider):
continue
elif provider.is_valid():
self.stdout.write(
self.style.SUCCESS(' # - %s "%s"' % (provider.name, provider))
)
elif verbosity >= 2:
self.stdout.write(
self.style.ERROR(
' # - %s "%s" (not found)' % (provider.name, provider)
)
)
provider = IniConfigProvider()
merger.write_provider(provider, include_doc=verbosity >= 2)
self.stdout.write(provider.to_str())
def show_python_config(self, verbosity):
self.stdout.write(self.style.SUCCESS("# " + "-" * 80))
self.stdout.write(
self.style.SUCCESS(
_("# Djangofloor version %(version)s") % {"version": version}
)
)
self.stdout.write(
self.style.SUCCESS(
_("# %(project)s version %(version)s")
% {
"version": guess_version(merger.settings),
"project": merger.settings["DF_PROJECT_NAME"],
}
)
)
self.stdout.write(self.style.SUCCESS("# Configuration providers:"))
for provider in merger.providers:
if provider.is_valid():
self.stdout.write(
self.style.SUCCESS('# - %s "%s"' % (provider.name, provider))
)
elif verbosity > 1:
self.stdout.write(
self.style.ERROR(
'# - %s "%s" (not found)' % (provider.name, provider)
)
)
self.stdout.write(self.style.SUCCESS("# " + "-" * 80))
setting_names = list(merger.raw_settings)
setting_names.sort()
# first, compute all imports to do
imports = {}
def add_import(val):
if not isinstance(val, type):
val = val.__class__
if val.__module__ != "builtins":
imports.setdefault(val.__module__, set()).add(val.__name__)
for setting_name in setting_names:
if setting_name not in merger.settings:
continue
value = merger.settings[setting_name]
add_import(value)
if imports:
self.stdout.write("\n")
for module_name in sorted(imports):
objects = ", ".join(sorted(imports[module_name]))
self.stdout.write(
self.style.WARNING("from %s import %s" % (module_name, objects))
)
self.stdout.write("\n")
for setting_name in setting_names:
if setting_name not in merger.settings:
continue
value = merger.settings[setting_name]
self.stdout.write(self.style.SUCCESS("%s = %r" % (setting_name, value)))
if verbosity <= 1:
continue
for provider_name, raw_value in merger.raw_settings[setting_name].items():
self.stdout.write(
self.style.WARNING(
" # %s -> %r" % (provider_name or "built-in", raw_value)
)
)
def show_heroku_config(self):
# Pipfile
# add extra packages (due to the config) to the Pipfile
# requirements.txt
# heroku addons:create heroku-postgresql:dev
queues = get_expected_queues()
self.stdout.write("web: %s-aiohttp" % settings.DF_MODULE_NAME)
for queue in queues:
self.stdout.write(
"%s: %s-%s worker -Q %s"
% (queue, settings.DF_MODULE_NAME, "celery", queue)
)
def show_social_auth_config(self):
from djangofloor.management.commands.social_authentications import (
Command as SACommand,
)
# noinspection PyCallByClass
SACommand.show_config(self)
class BaseCommand(DjBaseCommand):
""" A less terrible base class for Djagno management commands.
Changes:
* Tracebacks are turned on by default
* Verbosity is controlled by a counter instead of a number (ex,
``-vvv`` instead of ``-v3``) and are used to set up logging levels
too (disable by setting ``logging_enabled = False``).
* Will override a named logger (see ``logging_handler_name``) with the
level and formatter settings from the command line (or add a new
stderr logger if no matching logger is found).
* Subclassess can set ``option_list`` directly - the base options are
stored in ``base_option_list`` and merged with ``get_option_list``.
* The ``self.log`` attribute will be set to a logger configured to use
the name from ``self.get_log_name()`` (which defaults to the fully
qualified name of the module containing the command).
* If ``handle`` returns an integer it will be treated as a numeric exit
status. If a ``str`` or ``unicode`` is returned it will be treated
"normally" and 0 will be returned.
* If ``handle`` raises an exception that exception will be logged
(unless ``self.log_exc`` is ``False``) and the command will exit
with a status of 1.
Example::
from dwdj.management.base import BaseCommand, make_option
class MyCommand(BaseCommand):
option_list = [
make_option("-f", "--foo", action="store_true"),
]
def handle(self, *args, **options):
if options.get("foo"):
return 1
return 0
"""
# For now, tell Django to use the legacy optparse option parsing. At some
# point we'll need to update all the option parsing so it's done the fancy
# new way with argparse, but that day is not today.
use_argparse = False
base_option_list = [
make_option('-q', '--quiet', action="store_const", const=-1, dest="verbosity"),
make_option('-v', '--verbose', action="count", default=0, dest="verbosity"),
make_option('--verbose-log', action="store_true", help=dedent("""
Use a more verbose logging format.
""")),
make_option('--settings', help=dedent("""
The Python path to a settings module, e.g.
"myproject.settings.main". If this isn\'t provided, the
DJANGO_SETTINGS_MODULE environment variable will be used.')
""")),
make_option('--pythonpath', help=dedent("""
A directory to add to the Python path, e.g.
"/home/djangoprojects/myproject".
""")),
make_option('--traceback', action='store_true', default=True, help=dedent("""
Log complete exception traceback, not just exception and message.
""")),
make_option('--no-traceback', action='store_false', dest="traceback", help=dedent("""
Log only exception messages, not complete tracebacks.
"""))
]
option_list = []
logging_enabled = True
logging_handler_name = "stderr"
logging_format = '%(levelname)s [%(name)s]: %(message)s'
logging_format_verbose = '%(asctime)s %(levelname)s [%(processName)s:%(threadName)s:%(name)s]: %(message)s'
log_exc = True
def get_option_list(self):
return self.base_option_list + self.option_list
def create_parser(self, prog_name, subcommand):
"""
Create and return the ``OptionParser`` which will be used to
parse the arguments to this command.
"""
parser = OptionParser(prog=prog_name,
usage=self.usage(subcommand),
version=self.get_version(),
option_list=self.get_option_list())
original_parse = parser.parse_args
parser.parse_args = partial(self._override_parse_args, parser,
original_parse)
return parser
def _override_parse_args(self, parser, original_parse, *argv, **kwargs):
options, args = original_parse(*argv, **kwargs)
self.logging_setup(options)
if options.verbosity < 0:
options.traceback = False
return (options, args)
def logging_setup(self, options):
verbosity = options.verbosity
level = (
logging.CRITICAL if verbosity < 0 else
logging.ERROR if verbosity == 0 else
logging.WARNING if verbosity == 1 else
logging.INFO if verbosity == 2 else
logging.DEBUG
)
format = (
self.logging_format if not options.verbose_log else
self.logging_format_verbose
)
logger, handler = self.logging_get_handler()
formatter = logging.Formatter(format)
handler.setFormatter(formatter)
if logger.level > level:
logger.setLevel(level)
handler.setLevel(level)
def get_log_name(self):
return type(self).__module__
@cached_property
def log(self):
return logging.getLogger(self.get_log_name())
def logging_get_handler(self):
logger = logging.getLogger("")
for handler in logger.handlers:
if handler.name == self.logging_handler_name:
return logger, handler
handler = logging.StreamHandler(sys.stderr)
handler.name = self.logging_handler_name
logger.addHandler(handler)
return logger, handler
def run_from_argv(self, argv):
"""
Set up any environment changes requested (e.g., Python path
and Django settings), then run this command. If the
command raises a ``CommandError``, intercept it and print it sensibly
to stderr.
"""
signal_pre_run.send(self, argv=argv)
parser = self.create_parser(argv[0], argv[1])
options, args = parser.parse_args(argv[2:])
handle_default_options(options)
try:
result = self.execute(*args, **options.__dict__)
except SystemExit as e:
signal_post_run.send(self, status=e.code)
raise
except BaseException:
result = self.handle_execute_exc(options)
if result is None:
result = 1
status = result or 0
signal_post_run.send(self, status=status)
sys.exit(status)
def handle_execute_exc(self, options):
if not self.log_exc:
return 1
if options.traceback:
self.log.exception("Exception running command:")
else:
exc_info = sys.exc_info()
self.log.error("%s: %s", exc_info[0].__name__, exc_info[1])
return 1
def execute(self, *args, **options):
"""
Try to execute this command, performing model validation if
needed (as controlled by the attribute
``self.requires_model_validation``, except if force-skipped).
"""
# Switch to English, because django-admin.py creates database content
# like permissions, and those shouldn't contain any translations.
# But only do this if we can assume we have a working settings file,
# because django.utils.translation requires settings.
saved_lang = None
self.stdout = OutputWrapper(options.get('stdout', sys.stdout))
self.stderr = OutputWrapper(options.get('stderr', sys.stderr), self.style.ERROR)
if self.can_import_settings:
from django.utils import translation
saved_lang = translation.get_language()
translation.activate('en-us')
try:
if getattr(self, "requires_model_validation", False) and not options.get('skip_validation'):
self.validate()
result = self.handle(*args, **options)
if isinstance(result, basestring):
if self.output_transaction:
# This needs to be imported here, because it relies on
# settings.
from django.db import connections, DEFAULT_DB_ALIAS
connection = connections[options.get('database', DEFAULT_DB_ALIAS)]
if connection.ops.start_transaction_sql():
self.stdout.write(self.style.SQL_KEYWORD(connection.ops.start_transaction_sql()))
self.stdout.write(result)
if self.output_transaction:
self.stdout.write('\n' + self.style.SQL_KEYWORD("COMMIT;"))
result = 0
return result
finally:
if saved_lang is not None:
translation.activate(saved_lang)
class ReportingOrgMaker(object):
NO_CANDIDATE = 'no candidate'
MULTIPLE_CANDIDATES = 'multiple candidates'
SYNC_NOT_SUPPORT = 'sync not support'
def __init__(self, options):
self.stdout = OutputWrapper(options.get('stdout', sys.stdout))
self.keyword_based_reporting_orgs = {
'WASH Alliance': 8,
'Connect4Change': 34,
'SRHR Alliance': 1043,
'WvW': 43,
'wvw2014': 43,
'wvw2015': 43,
'wvw2016': 43,
'WfW': 43,
'wfw2014': 43,
'wfw2015': 43,
'wfw2016': 43,
}
self.keywords_set = set(self.keyword_based_reporting_orgs.keys())
self.migrate = options['migrate']
self.ok_list = []
self.fix_list = []
def add_to_ok(self, project, org):
self.ok_list += [OKProject(project, org)]
def add_to_fix(self, project, reason, partners, sync_owner=None):
self.fix_list += [FixProject(project, reason, partners, sync_owner)]
def find_reporting_org_for_projects(self):
# loop over all projects, trying ot figure reporting-org for each.
self.stdout.write('\nData gathering progress:')
i = 1
for project in Project.objects.published().prefetch_related(
'partnerships', 'partnerships__organisation', 'keywords'):
if not i % 100:
self.stdout.write(str(i))
else:
self.stdout.write(".", ending='')
i += 1
self.stdout.flush()
# first check if we have a keyword from the keyword_based_reporting_orgs.keys() list
reporting_keyword = self.keywords_set.intersection(
set(project.keywords.values_list('label', flat=True)))
if reporting_keyword:
# if we do, set the reporting-org to the org connected to the keyword
self.add_to_ok(project, Organisation.objects.get(
pk=self.keyword_based_reporting_orgs[list(reporting_keyword)[0]]))
else:
# otherwise try to find the reporting org among sync_owner and accountable partners
support_partners = project.partnerships.filter(
iati_organisation_role=Partnership.IATI_ACCOUNTABLE_PARTNER
).select_related('organisation')
# If there's no support partner, we set the sync_owner as reporting-org,
# if there is one. Otherwise we report the problem.
if support_partners.count() == 0:
if project.sync_owner:
self.add_to_ok(project, project.sync_owner)
else:
self.add_to_fix(project, self.NO_CANDIDATE, [])
# If we have exactly one support partner, then things are in order if either:
# 1) the sync_owner matches the support partner
# 2) there is no sync_owner
# In both cases we should be fine to set the sync_owner/support partner as the
# reporting-org.
elif support_partners.count() == 1:
if project.sync_owner:
# 1)
if project.sync_owner == support_partners[0].organisation:
self.add_to_ok(project, project.sync_owner)
else:
self.add_to_fix(project, self.SYNC_NOT_SUPPORT, support_partners,
project.sync_owner)
# 2)
else:
self.add_to_ok(project, support_partners[0].organisation)
# If there are multiple support partners we check if one of the partners is sync_owner
# we set that organisation to reporting. Otherwise we report the problem.
else:
if project.sync_owner:
if project.sync_owner.id in [p.organisation.id for p in support_partners]:
self.add_to_ok(project, project.sync_owner)
else:
self.add_to_fix(project, self.MULTIPLE_CANDIDATES, support_partners)
def create_reporting_orgs(self):
try:
reporting_org_choice = Partnership.IATI_REPORTING_ORGANISATION
self.stdout.write(
u"\n*** Assigning reporting-org partners to the following projects ***"
)
self.stdout.write(
u"project ID, project title, organisation id, organisation name"
)
for data in self.ok_list:
partner = Partnership(
organisation=data.organisation,
iati_organisation_role=reporting_org_choice)
data.project.partnerships.add(partner)
self.print_ok_data(data)
except:
self.stdout.write(
u"\n*** Reporting organisation choice not available for Partnerships ***"
)
def print_ok_data(self, data):
self.stdout.write(
u'{},"{}",{},"{}"'.format(data.project.id, data.project.title, data.organisation.id,
data.organisation.name))
def print_fix_data(self, data, partner):
self.stdout.write(
u'{},"{}",{},"{}","{}",{},"{}"'.format(
data.project.id,
data.project.title,
partner.organisation.id,
partner.organisation.name,
data.reason,
data.sync_owner.id if data.sync_owner else '',
data.sync_owner.name if data.sync_owner else ''))
def output_ok_list(self):
self.stdout.write(
u"\n*** List of projects and the <reporting-org> partner they will get when migrating ***"
)
self.stdout.write(
u"project ID, project title, organisation id, organisation name"
)
for data in self.ok_list:
self.print_ok_data(data)
def output_fix_list(self):
self.stdout.write(
u"\n*** List of projects where no clear-cut reporting-org candidate was found ***"
)
self.stdout.write(
u"project ID, project title, support partner id, support partner name, type of problem, sync_owner id, sync_owner name"
)
for data in self.fix_list:
for partner in data.partners:
self.print_fix_data(data, partner)
class LoggingMixin:
verbosity = 3 if settings.DEBUG else 1
"""@type: int"""
indent = 0
"""@type: int"""
logging_level = logging.DEBUG if settings.DEBUG else 1
log_map = dict()
logging_initialized = False
print_level = True
def set_verbosity(self, verbosity):
LEVELS = {
0: logging.CRITICAL,
1: logging.ERROR,
2: logging.WARNING,
3: logging.DEBUG,
}
self.verbosity = verbosity
self.logging_level = LEVELS[verbosity]
def initialize_logging(self):
if not self.logging_initialized:
try:
self.stdout = OutputWrapper(self.stdout._out, ending='')
except AttributeError:
self.stdout = OutputWrapper(sys.stdout, ending='')
# self.stdout = codecs.getwriter('utf8')(self.stdout)
self.logging_initialized = True
def color_format(self, level, message):
level_colors = {
# Level and a pair of colors: first for the label,
# the rest for the text;
# the bolder color label can make them easier to spot
# in the console log.
logging.DEBUG: (33, 39),
# logging.TRACE: (147, 153),
logging.INFO: (43, 49),
logging.WARNING: (214, 226),
logging.ERROR: (196, 197),
logging.CRITICAL: (196, 197),
}.get(level, (33, 39))
# 256-color to give wider spectrum than just ANSI
color = "\033[38;5;{:d}m"
reset = "\033[0m"
# Pass any simple messages from internal things, like Django's
# runserver, without special formatting.
mp_levels = {
logging.INFO: u'INF',
logging.WARNING: u'WRN',
logging.ERROR: u'ERR',
logging.DEBUG: u'DBG',
logging.CRITICAL: u'CRT'
}
st_level = mp_levels[level]
level_prefix = '%s[%s] ' % (color.format(level_colors[0]), st_level)
return u'{level_prefix}{color_normal}{message}{reset}'.format(
level_prefix=level_prefix if self.print_level else '',
message=message,
color_normal=color.format(level_colors[1]),
reset=reset
)
def llog(self, logging_level, format_string, *args):
"""
@param logging_level:
50 = summary/critical
40 = error
30 = warning
20 = info
10 = debug
@return:
"""
LEVELS = {
logging.CRITICAL,
logging.ERROR,
logging.WARNING,
logging.INFO,
logging.DEBUG
}
if logging_level not in LEVELS:
logging_level = logging.DEBUG
message = format_string % args
if logging_level >= self.logging_level:
if hasattr(self, 'stdout'):
self.initialize_logging()
self.stdout.write(u' ' * self.indent)
if self.stdout.isatty():
self.stdout.write(self.color_format(
logging_level, message))
else:
self.stdout.write(message)
self.stdout.write('\n')
self.log_map.setdefault(logging_level, []).append(message)
def log(self, format_string, *args):
message = format_string % args
if hasattr(self, 'stdout'):
self.initialize_logging()
self.stdout.write(u' ' * self.indent)
self.stdout.write(message)
self.stdout.write('\n')
def critical(self, format_string, *args):
self.llog(logging.CRITICAL, format_string, *args)
def debug(self, format_string, *args):
self.llog(logging.DEBUG, format_string, *args)
def info(self, format_string, *args):
self.llog(logging.INFO, format_string, *args)
def warning(self, format_string, *args):
self.llog(logging.WARNING, format_string, *args)
def error(self, format_string, *args):
self.llog(logging.ERROR, format_string, *args)
def exception(self, format_string, *args):
p_type, p_exception, _ = sys.exc_info()
self.llog(logging.ERROR, format_string, *args)
self.llog(logging.ERROR, u'Exception message: %s', p_exception)
self.llog(logging.ERROR, u'Exception type : %s', p_type)
self.llog(logging.ERROR, u'Traceback\n%s', format_exc())
