class Command(BaseCommand): def __init__(self, *args, **kwargs): from djangae.boot import setup_paths, setup_datastore_stubs setup_paths() super(Command, self).__init__(*args, **kwargs) def run_from_argv(self, argv): from google.appengine.ext.remote_api import remote_api_stub from google.appengine.tools import appengine_rpc import getpass from djangae.boot import find_project_root self.stdout = OutputWrapper(sys.stdout) def auth_func(): return (raw_input('Google Account Login:'******'Password:'******'app.yaml')).read() app_id = app_yaml.split("application:")[1].lstrip().split()[0] self.stdout.write("Opening Remote API connection to {0}...\n".format(app_id)) remote_api_stub.ConfigureRemoteApi(None, '/_ah/remote_api', auth_func, servername='{0}.appspot.com'.format(app_id), secure=True, ) self.stdout.write("...Connection established...have a nice day :)\n".format(app_id)) argv = argv[:1] + argv[2:] execute_from_command_line(argv)
def run_migrations(args, options, executor_codename, schema_name, allow_atomic=True): from django.core.management import color from django.core.management.base import OutputWrapper from django.db import connection style = color.color_style() def style_func(msg): return '[%s:%s] %s' % ( style.NOTICE(executor_codename), style.NOTICE(schema_name), msg ) connection.set_schema(schema_name) stdout = OutputWrapper(sys.stdout) stdout.style_func = style_func stderr = OutputWrapper(sys.stderr) stderr.style_func = style_func if int(options.get('verbosity', 1)) >= 1: stdout.write(style.NOTICE("=== Starting migration")) MigrateCommand(stdout=stdout, stderr=stderr).execute(*args, **options) try: transaction.commit() connection.close() connection.connection = None except transaction.TransactionManagementError: if not allow_atomic: raise # We are in atomic transaction, don't close connections pass connection.set_schema_to_public()
class Command(BaseCommand): def __init__(self, *args, **kwargs): from djangae.boot import setup_paths, setup_datastore_stubs setup_paths() super(Command, self).__init__(*args, **kwargs) def run_from_argv(self, argv): from google.appengine.ext.remote_api import remote_api_stub from google.appengine.tools import appengine_rpc import getpass from djangae.boot import find_project_root self.stdout = OutputWrapper(sys.stdout) def auth_func(): return (raw_input('Google Account Login:'******'Password:'******'app.yaml')).read() app_id = app_yaml.split("application:")[1].lstrip().split()[0] self.stdout.write( "Opening Remote API connection to {0}...\n".format(app_id)) remote_api_stub.ConfigureRemoteApi( None, '/_ah/remote_api', auth_func, servername='{0}.appspot.com'.format(app_id), secure=True, ) self.stdout.write( "...Connection established...have a nice day :)\n".format(app_id)) argv = argv[:1] + argv[2:] execute_from_command_line(argv)
def multiprocess_reader(urls, stdout=None): stdout = OutputWrapper(stdout or sys.stdout) result = URLReader(urls=urls)() out = set() for built_result in result: out.add(built_result) stdout.write("Read {}".format(built_result.url)) return out
def write(msg, is_error=False): stdout = OutputWrapper(sys.stdout) style = color_style() if is_error: styling_msg = style.ERROR(msg) else: styling_msg = style.SUCCESS(msg) stdout.write(styling_msg)
def run_migrations(args, options, executor_codename, schema_name, tenant_type='', allow_atomic=True, idx=None, count=None): from django.core.management import color from django.core.management.base import OutputWrapper from django.db import connections style = color.color_style() def style_func(msg): percent_str = '' if idx is not None and count is not None and count > 0: percent_str = '%d/%d (%s%%) ' % (idx + 1, count, int(100 * (idx + 1) / count)) message = '[%s%s:%s] %s' % ( percent_str, style.NOTICE(executor_codename), style.NOTICE(schema_name), msg ) signal_message = '[%s%s:%s] %s' % ( percent_str, executor_codename, schema_name, msg ) schema_migrate_message.send(run_migrations, message=signal_message) return message connection = connections[options.get('database', get_tenant_database_alias())] connection.set_schema(schema_name, tenant_type=tenant_type) # ensure that django_migrations table is created in the schema before migrations run, otherwise the migration # table in the public schema gets picked and no migrations are applied migration_recorder = MigrationRecorder(connection) migration_recorder.ensure_schema() stdout = OutputWrapper(sys.stdout) stdout.style_func = style_func stderr = OutputWrapper(sys.stderr) stderr.style_func = style_func if int(options.get('verbosity', 1)) >= 1: stdout.write(style.NOTICE("=== Starting migration")) MigrateCommand(stdout=stdout, stderr=stderr).execute(*args, **options) try: transaction.commit() connection.close() connection.connection = None except transaction.TransactionManagementError: if not allow_atomic: raise # We are in atomic transaction, don't close connections pass connection.set_schema_to_public() schema_migrated.send(run_migrations, schema_name=schema_name)
def join_individual(self, b: 'Individual', out: OutputWrapper = None): if out: out.write("Карт для переноса: %s" % Card.objects.filter(individual=b).count()) slog.Log(key=str(self.pk), type=2002, body=simplejson.dumps({"Сохраняемая запись": str(self), "Объединяемая запись": str(b)}), user=None).save() for c in Card.objects.filter(individual=b): c.individual = self c.save() b.delete()
def multiprocess_writer(data, stdout=None): stdout = OutputWrapper(stdout or sys.stdout) result = URLWriter(data=data)() out = set() for built_result in result: out.add(built_result) if built_result.created: stdout.write("Created {}".format(built_result.name)) elif built_result.modified: stdout.write("Updated {}".format(built_result.name)) return out
class Log: def __init__(self): self._output = OutputWrapper(stdout) def print(self, msg, ending='\n', color_prefix=''): self._output.write(f'{color_prefix}{msg}\033[0m', ending=ending) def printerr(self, msg, ending='\n'): self.print(msg, ending, color_prefix='\033[91m') def printok(self, msg, ending='\n'): self.print(msg, ending, color_prefix='\033[92m')
def execute(self, *args, **options): try: super(Command, self).execute(*args, **options) except CommandError as e: if options.get('traceback', False): raise # self.stderr is not guaranteed to be set here stderr = getattr(self, 'stderr', None) if not stderr: stderr = OutputWrapper(sys.stderr, self.style.ERROR) stderr.write('%s: %s' % (e.__class__.__name__, e)) sys.exit(2)
def main(): parser = OptionParser(usage=textwrap.dedent("""\ %prog [options] <instance-name> [<target-directory>] Creates a new EOxServer instance with all necessary files and folder structure. Optionally, a SQLite database is initiated. """), version=eoxserver.get_version()) parser.add_option('-i', '--init-spatialite', '--init_spatialite', dest='init_spatialite', action='store_true', default=False, help='Flag to initialize the sqlite database.') parser.add_option('-v', '--verbosity', action='store', dest='verbosity', default='1', type='choice', choices=['0', '1', '2', '3']) parser.add_option('--traceback', action='store_true', help='Raise on exception') options, args = parser.parse_args() error_stream = OutputWrapper(sys.stderr, color_style().ERROR) if not args: error_stream.write("Mandatory argument 'instance-name' not given.\n") sys.exit(1) name = args[0] try: target = args[1] except IndexError: target = None try: create_instance(name, target, **options.__dict__) except Exception as e: if options.traceback: raise error_stream.write("%s: %s\n" % (e.__class__.__name__, e))
def run_migrations(args, options, executor_codename, schema_name, allow_atomic=True, idx=None, count=None): from django.core.management import color from django.core.management.base import OutputWrapper from django.db import connections style = color.color_style() def style_func(msg): percent_str = '' if idx is not None and count is not None and count > 0: percent_str = '%d/%d (%s%%) ' % (idx + 1, count, int(100 * (idx + 1) / count)) return '[%s%s:%s] %s' % (percent_str, style.NOTICE(executor_codename), style.NOTICE(schema_name), msg) include_public = True if (options.get('shared') or schema_name == 'public') else False connection = connections[get_tenant_database_alias()] connection.set_schema(schema_name, include_public=include_public) stdout = OutputWrapper(sys.stdout) stdout.style_func = style_func stderr = OutputWrapper(sys.stderr) stderr.style_func = style_func if int(options.get('verbosity', 1)) >= 1: stdout.write(style.NOTICE("=== Starting migration")) MigrateCommand(stdout=stdout, stderr=stderr).execute(*args, **options) try: transaction.commit() connection.close() connection.connection = None except transaction.TransactionManagementError: if not allow_atomic: raise # We are in atomic transaction, don't close connections pass connection.set_schema_to_public()
def run_migrations(args, options, executor_codename, schema_name, allow_atomic=True): from django.core.management import color from django.core.management.base import OutputWrapper from django.db import connections PUBLIC_SCHEMA_NAME = get_public_schema_name() options['database'] = settings.TENANT_DATABASE if schema_name == PUBLIC_SCHEMA_NAME: options['database'] = DEFAULT_DB_ALIAS style = color.color_style() def style_func(msg): return '[%s:%s:%s] %s' % (options['database'], style.NOTICE(executor_codename), style.NOTICE(schema_name), msg) connections[options['database']].set_schema(schema_name) stdout = OutputWrapper(sys.stdout) stdout.style_func = style_func stderr = OutputWrapper(sys.stderr) stderr.style_func = style_func if int(options.get('verbosity', 1)) >= 1: stdout.write(style.NOTICE("=== Starting migration")) MigrateCommand(stdout=stdout, stderr=stderr).execute(*args, **options) try: transaction.commit() connections[options['database']].close() connections[options['database']].connection = None except transaction.TransactionManagementError: if not allow_atomic: raise # We are in atomic transaction, don't close connections pass
def main(): parser = OptionParser( usage=textwrap.dedent("""\ %prog [options] <instance-name> [<target-directory>] Creates a new EOxServer instance with all necessary files and folder structure. Optionally, a SQLite database is initiated. """), version=eoxserver.get_version() ) parser.add_option('-i', '--init-spatialite', '--init_spatialite', dest='init_spatialite', action='store_true', default=False, help='Flag to initialize the sqlite database.' ) parser.add_option('-v', '--verbosity', action='store', dest='verbosity', default='1', type='choice', choices=['0', '1', '2', '3'] ) parser.add_option('--traceback', action='store_true', help='Raise on exception' ) options, args = parser.parse_args() error_stream = OutputWrapper(sys.stderr, color_style().ERROR) if not args: error_stream.write("Mandatory argument 'instance-name' not given.\n") sys.exit(1) name = args[0] try: target = args[1] except IndexError: target = None try: create_instance(name, target, **options.__dict__) except Exception as e: if options.traceback: raise error_stream.write("%s: %s\n" % (e.__class__.__name__, e))
def run_migrations(args, options, executor_codename, schema_name, allow_atomic=True, idx=None, count=None): from django.core.management import color from django.core.management.base import OutputWrapper from django.db import connections style = color.color_style() def style_func(msg): percent_str = '' if idx is not None and count is not None and count > 0: percent_str = '%d/%d (%s%%) ' % (idx + 1, count, int(100 * (idx + 1) / count)) return '[%s%s:%s] %s' % ( percent_str, style.NOTICE(executor_codename), style.NOTICE(schema_name), msg ) connection = connections[get_tenant_database_alias()] connection.set_schema(schema_name) stdout = OutputWrapper(sys.stdout) stdout.style_func = style_func stderr = OutputWrapper(sys.stderr) stderr.style_func = style_func if int(options.get('verbosity', 1)) >= 1: stdout.write(style.NOTICE("=== Starting migration")) MigrateCommand(stdout=stdout, stderr=stderr).execute(*args, **options) try: transaction.commit() connection.close() connection.connection = None except transaction.TransactionManagementError: if not allow_atomic: raise # We are in atomic transaction, don't close connections pass connection.set_schema_to_public()
class ImportData: def __init__(self): self.stdout = OutputWrapper(sys.stdout) self.stderr = OutputWrapper(sys.stderr) def init_sampledataset(self, target_lists): """ :param target_lists: :return: """ self.stdout.write('Loading data...') for file in target_lists: self.stdout.write(f'file: {file}') with open(file) as f: reader = csv.reader(f) # skip header next(reader, None) for row in reader: data = { 'date': datetime.strptime(row[0], '%d.%m.%Y').date(), 'channel': row[1], 'country': row[2], 'os': row[3], 'impressions': row[4], 'clicks': row[5], 'installs': row[6], 'spend': Decimal(str(row[7])), 'revenue': Decimal(str(row[8])), } SampleDataset.objects.update_or_create(**data) self.stdout.write('Done!')
def on_execute_command(self, tenant, args, options): style = color.color_style() stdout = OutputWrapper(sys.stdout) stdout.write(style.MIGRATE_HEADING("=".ljust(70, "="))) stdout.write(style.MIGRATE_HEADING("=== Starting collectstatic: {0} ".format(tenant.schema_name).ljust(70, "="))) stdout.write(style.MIGRATE_HEADING("=".ljust(70, "="))) options["interactive"] = False super(Command, self).on_execute_command(tenant, args, options)
def run_migrations(args, options, executor_codename, schema_name, allow_atomic=True): from django.core.management import color from django.core.management.base import OutputWrapper from django.db import connection style = color.color_style() def style_func(msg): return '[%s:%s] %s' % (style.NOTICE(executor_codename), style.NOTICE(schema_name), msg) stdout = OutputWrapper(sys.stdout) stdout.style_func = style_func stderr = OutputWrapper(sys.stderr) stderr.style_func = style_func if int(options.get('verbosity', 1)) >= 1: stdout.write( style.NOTICE("=== Running migrate for schema %s" % schema_name)) connection.set_schema(schema_name) MigrateCommand(stdout=stdout, stderr=stderr).execute(*args, **options) try: transaction.commit() connection.close() connection.connection = None except transaction.TransactionManagementError: if not allow_atomic: raise # We are in atomic transaction, don't close connections pass connection.set_schema_to_public()
def on_execute_command(self, tenant, args, options): style = color.color_style() stdout = OutputWrapper(sys.stdout) stdout.write(style.MIGRATE_HEADING("=".ljust(70, "="))) stdout.write( style.MIGRATE_HEADING("=== Starting collectstatic: {0} ".format( tenant.schema_name).ljust(70, "="))) stdout.write(style.MIGRATE_HEADING("=".ljust(70, "="))) options["interactive"] = False super(Command, self).on_execute_command(tenant, args, options)
class ReportingOrgMaker(object): NO_CANDIDATE = 'no candidate' MULTIPLE_CANDIDATES = 'multiple candidates' SYNC_NOT_SUPPORT = 'sync not support' def __init__(self, options): self.stdout = OutputWrapper(options.get('stdout', sys.stdout)) self.keyword_based_reporting_orgs = { 'WASH Alliance': 8, 'Connect4Change': 34, 'SRHR Alliance': 1043, 'WvW': 43, 'wvw2014': 43, 'wvw2015': 43, 'wvw2016': 43, 'WfW': 43, 'wfw2014': 43, 'wfw2015': 43, 'wfw2016': 43, } self.keywords_set = set(self.keyword_based_reporting_orgs.keys()) self.migrate = options['migrate'] self.ok_list = [] self.fix_list = [] def add_to_ok(self, project, org): self.ok_list += [OKProject(project, org)] def add_to_fix(self, project, reason, partners, sync_owner=None): self.fix_list += [FixProject(project, reason, partners, sync_owner)] def find_reporting_org_for_projects(self): # loop over all projects, trying ot figure reporting-org for each. self.stdout.write('\nData gathering progress:') i = 1 for project in Project.objects.published().prefetch_related( 'partnerships', 'partnerships__organisation', 'keywords'): if not i % 100: self.stdout.write(str(i)) else: self.stdout.write(".", ending='') i += 1 self.stdout.flush() # first check if we have a keyword from the keyword_based_reporting_orgs.keys() list reporting_keyword = self.keywords_set.intersection( set(project.keywords.values_list('label', flat=True))) if reporting_keyword: # if we do, set the reporting-org to the org connected to the keyword self.add_to_ok(project, Organisation.objects.get( pk=self.keyword_based_reporting_orgs[list(reporting_keyword)[0]])) else: # otherwise try to find the reporting org among sync_owner and accountable partners support_partners = project.partnerships.filter( iati_organisation_role=Partnership.IATI_ACCOUNTABLE_PARTNER ).select_related('organisation') # If there's no support partner, we set the sync_owner as reporting-org, # if there is one. Otherwise we report the problem. if support_partners.count() == 0: if project.sync_owner: self.add_to_ok(project, project.sync_owner) else: self.add_to_fix(project, self.NO_CANDIDATE, []) # If we have exactly one support partner, then things are in order if either: # 1) the sync_owner matches the support partner # 2) there is no sync_owner # In both cases we should be fine to set the sync_owner/support partner as the # reporting-org. elif support_partners.count() == 1: if project.sync_owner: # 1) if project.sync_owner == support_partners[0].organisation: self.add_to_ok(project, project.sync_owner) else: self.add_to_fix(project, self.SYNC_NOT_SUPPORT, support_partners, project.sync_owner) # 2) else: self.add_to_ok(project, support_partners[0].organisation) # If there are multiple support partners we check if one of the partners is sync_owner # we set that organisation to reporting. Otherwise we report the problem. else: if project.sync_owner: if project.sync_owner.id in [p.organisation.id for p in support_partners]: self.add_to_ok(project, project.sync_owner) else: self.add_to_fix(project, self.MULTIPLE_CANDIDATES, support_partners) def create_reporting_orgs(self): try: reporting_org_choice = Partnership.IATI_REPORTING_ORGANISATION self.stdout.write( u"\n*** Assigning reporting-org partners to the following projects ***" ) self.stdout.write( u"project ID, project title, organisation id, organisation name" ) for data in self.ok_list: partner = Partnership( organisation=data.organisation, iati_organisation_role=reporting_org_choice) data.project.partnerships.add(partner) self.print_ok_data(data) except: self.stdout.write( u"\n*** Reporting organisation choice not available for Partnerships ***" ) def print_ok_data(self, data): self.stdout.write( u'{},"{}",{},"{}"'.format(data.project.id, data.project.title, data.organisation.id, data.organisation.name)) def print_fix_data(self, data, partner): self.stdout.write( u'{},"{}",{},"{}","{}",{},"{}"'.format( data.project.id, data.project.title, partner.organisation.id, partner.organisation.name, data.reason, data.sync_owner.id if data.sync_owner else '', data.sync_owner.name if data.sync_owner else '')) def output_ok_list(self): self.stdout.write( u"\n*** List of projects and the <reporting-org> partner they will get when migrating ***" ) self.stdout.write( u"project ID, project title, organisation id, organisation name" ) for data in self.ok_list: self.print_ok_data(data) def output_fix_list(self): self.stdout.write( u"\n*** List of projects where no clear-cut reporting-org candidate was found ***" ) self.stdout.write( u"project ID, project title, support partner id, support partner name, type of problem, sync_owner id, sync_owner name" ) for data in self.fix_list: for partner in data.partners: self.print_fix_data(data, partner)
class ReportingOrgMaker(object): NO_CANDIDATE = 'no candidate' MULTIPLE_CANDIDATES = 'multiple candidates' SYNC_NOT_SUPPORT = 'sync not support' def __init__(self, options): self.stdout = OutputWrapper(options.get('stdout', sys.stdout)) self.keyword_based_reporting_orgs = { 'WASH Alliance': 8, 'Connect4Change': 34, 'SRHR Alliance': 1043, 'WvW': 43, 'wvw2014': 43, 'wvw2015': 43, 'wvw2016': 43, 'WfW': 43, 'wfw2014': 43, 'wfw2015': 43, 'wfw2016': 43, } self.keywords_set = set(self.keyword_based_reporting_orgs.keys()) self.migrate = options['migrate'] self.ok_list = [] self.fix_list = [] def add_to_ok(self, project, org): self.ok_list += [OKProject(project, org)] def add_to_fix(self, project, reason, partners, sync_owner=None): self.fix_list += [FixProject(project, reason, partners, sync_owner)] def find_reporting_org_for_projects(self): # loop over all projects, trying ot figure reporting-org for each. self.stdout.write('\nData gathering progress:') i = 1 for project in Project.objects.published().prefetch_related( 'partnerships', 'partnerships__organisation', 'keywords'): if not i % 100: self.stdout.write(str(i)) else: self.stdout.write(".", ending='') i += 1 self.stdout.flush() # first check if we have a keyword from the keyword_based_reporting_orgs.keys() list reporting_keyword = self.keywords_set.intersection( set(project.keywords.values_list('label', flat=True))) if reporting_keyword: # if we do, set the reporting-org to the org connected to the keyword self.add_to_ok( project, Organisation.objects.get( pk=self.keyword_based_reporting_orgs[list( reporting_keyword)[0]])) else: # otherwise try to find the reporting org among sync_owner and accountable partners support_partners = project.partnerships.filter( iati_organisation_role=Partnership.IATI_ACCOUNTABLE_PARTNER ).select_related('organisation') # If there's no support partner, we set the sync_owner as reporting-org, # if there is one. Otherwise we report the problem. if support_partners.count() == 0: if project.sync_owner: self.add_to_ok(project, project.sync_owner) else: self.add_to_fix(project, self.NO_CANDIDATE, []) # If we have exactly one support partner, then things are in order if either: # 1) the sync_owner matches the support partner # 2) there is no sync_owner # In both cases we should be fine to set the sync_owner/support partner as the # reporting-org. elif support_partners.count() == 1: if project.sync_owner: # 1) if project.sync_owner == support_partners[ 0].organisation: self.add_to_ok(project, project.sync_owner) else: self.add_to_fix(project, self.SYNC_NOT_SUPPORT, support_partners, project.sync_owner) # 2) else: self.add_to_ok(project, support_partners[0].organisation) # If there are multiple support partners we check if one of the partners is sync_owner # we set that organisation to reporting. Otherwise we report the problem. else: if project.sync_owner: if project.sync_owner.id in [ p.organisation.id for p in support_partners ]: self.add_to_ok(project, project.sync_owner) else: self.add_to_fix(project, self.MULTIPLE_CANDIDATES, support_partners) def create_reporting_orgs(self): try: reporting_org_choice = Partnership.IATI_REPORTING_ORGANISATION self.stdout.write( u"\n*** Assigning reporting-org partners to the following projects ***" ) self.stdout.write( u"project ID, project title, organisation id, organisation name" ) for data in self.ok_list: partner = Partnership( organisation=data.organisation, iati_organisation_role=reporting_org_choice) data.project.partnerships.add(partner) self.print_ok_data(data) except: self.stdout.write( u"\n*** Reporting organisation choice not available for Partnerships ***" ) def print_ok_data(self, data): self.stdout.write(u'{},"{}",{},"{}"'.format(data.project.id, data.project.title, data.organisation.id, data.organisation.name)) def print_fix_data(self, data, partner): self.stdout.write(u'{},"{}",{},"{}","{}",{},"{}"'.format( data.project.id, data.project.title, partner.organisation.id, partner.organisation.name, data.reason, data.sync_owner.id if data.sync_owner else '', data.sync_owner.name if data.sync_owner else '')) def output_ok_list(self): self.stdout.write( u"\n*** List of projects and the <reporting-org> partner they will get when migrating ***" ) self.stdout.write( u"project ID, project title, organisation id, organisation name") for data in self.ok_list: self.print_ok_data(data) def output_fix_list(self): self.stdout.write( u"\n*** List of projects where no clear-cut reporting-org candidate was found ***" ) self.stdout.write( u"project ID, project title, support partner id, support partner name, type of problem, sync_owner id, sync_owner name" ) for data in self.fix_list: for partner in data.partners: self.print_fix_data(data, partner)
class Command(BaseCommand): help = ( "show the current configuration." 'Can display as python file ("config python") or as .ini file ("config ini"). Use -v 2 to display more info.' ) requires_system_checks = False options = { "python": "display the current config as Python module", "ini": "display the current config as .ini file", "heroku": "display a configuration valid to deploy on Heroku", "apache": "display an example of Apache config", "nginx": "display an example of Nginx config", "systemd": "display an example of systemd config", "supervisor": "display an example of Supervisor config", "social_authentications": "display configured social authentications", } if settings.USE_CELERY: options["signals"] = "show the defined signals and remote functions" def add_arguments(self, parser): assert isinstance(parser, ArgumentParser) parser.add_argument( "action", default="show", choices=self.options, help=",\n".join(['"%s": %s' % x for x in self.options.items()]), ) parser.add_argument( "--filename", default=None, help="write output to this file" ) remove_arguments_from_help( parser, {"--settings", "--traceback", "--pythonpath"} ) def handle(self, *args, **options): try: self.handle_head(**options) except BrokenPipeError: pass def handle_head(self, **options): action = options["action"] verbosity = options["verbosity"] filename = options["filename"] if filename: self.stdout = OutputWrapper(open(filename, "w")) self.style = no_style() if action == "python": self.show_python_config(verbosity) elif action == "ini": self.show_ini_config(verbosity) elif action == "signals": self.show_signals_config() elif action == "heroku": self.show_heroku_config() elif action == "apache": self.show_external_config("djangofloor/config/apache.conf") elif action == "nginx": self.show_external_config("djangofloor/config/nginx.conf") elif action == "systemd": self.show_external_config("djangofloor/config/systemd.conf") elif action == "supervisor": self.show_external_config("djangofloor/config/supervisor.conf") elif action == "social_authentications": self.show_social_auth_config() def show_external_config(self, config): content = render_to_string(config, merger.settings) self.stdout.write(content) def show_signals_config(self): import_signals_and_functions() def display_callable(conn): fn = conn.function if getattr(fn, "__module__", None) and getattr(fn, "__name__", None): path = "%s.%s" % (fn.__module__, fn.__name__) elif getattr(fn, "__name__", None): path = fn.__name__ else: path = str(fn) return path self.stdout.write(self.style.ERROR("Signals")) data = list(decorators.REGISTERED_SIGNALS.items()) for name, connections in sorted(data, key=lambda x: x[0]): self.stdout.write(self.style.WARNING(' "%s"' % name)) for connection in connections: self.stdout.write( self.style.NOTICE(" -> %s" % display_callable(connection)) ) self.stdout.write(self.style.ERROR("Functions")) data = list(decorators.REGISTERED_FUNCTIONS.items()) for name, connection in sorted(data, key=lambda x: x[0]): self.stdout.write( self.style.WARNING( ' "%s" -> %s' % (name, display_callable(connection)) ) ) def show_ini_config(self, verbosity): if verbosity >= 2: self.stdout.write(self.style.SUCCESS("# read configuration files:")) for provider in merger.providers: if not isinstance(provider, IniConfigProvider): continue elif provider.is_valid(): self.stdout.write( self.style.SUCCESS(' # - %s "%s"' % (provider.name, provider)) ) elif verbosity >= 2: self.stdout.write( self.style.ERROR( ' # - %s "%s" (not found)' % (provider.name, provider) ) ) provider = IniConfigProvider() merger.write_provider(provider, include_doc=verbosity >= 2) self.stdout.write(provider.to_str()) def show_python_config(self, verbosity): self.stdout.write(self.style.SUCCESS("# " + "-" * 80)) self.stdout.write( self.style.SUCCESS( _("# Djangofloor version %(version)s") % {"version": version} ) ) self.stdout.write( self.style.SUCCESS( _("# %(project)s version %(version)s") % { "version": guess_version(merger.settings), "project": merger.settings["DF_PROJECT_NAME"], } ) ) self.stdout.write(self.style.SUCCESS("# Configuration providers:")) for provider in merger.providers: if provider.is_valid(): self.stdout.write( self.style.SUCCESS('# - %s "%s"' % (provider.name, provider)) ) elif verbosity > 1: self.stdout.write( self.style.ERROR( '# - %s "%s" (not found)' % (provider.name, provider) ) ) self.stdout.write(self.style.SUCCESS("# " + "-" * 80)) setting_names = list(merger.raw_settings) setting_names.sort() # first, compute all imports to do imports = {} def add_import(val): if not isinstance(val, type): val = val.__class__ if val.__module__ != "builtins": imports.setdefault(val.__module__, set()).add(val.__name__) for setting_name in setting_names: if setting_name not in merger.settings: continue value = merger.settings[setting_name] add_import(value) if imports: self.stdout.write("\n") for module_name in sorted(imports): objects = ", ".join(sorted(imports[module_name])) self.stdout.write( self.style.WARNING("from %s import %s" % (module_name, objects)) ) self.stdout.write("\n") for setting_name in setting_names: if setting_name not in merger.settings: continue value = merger.settings[setting_name] self.stdout.write(self.style.SUCCESS("%s = %r" % (setting_name, value))) if verbosity <= 1: continue for provider_name, raw_value in merger.raw_settings[setting_name].items(): self.stdout.write( self.style.WARNING( " # %s -> %r" % (provider_name or "built-in", raw_value) ) ) def show_heroku_config(self): # Pipfile # add extra packages (due to the config) to the Pipfile # requirements.txt # heroku addons:create heroku-postgresql:dev queues = get_expected_queues() self.stdout.write("web: %s-aiohttp" % settings.DF_MODULE_NAME) for queue in queues: self.stdout.write( "%s: %s-%s worker -Q %s" % (queue, settings.DF_MODULE_NAME, "celery", queue) ) def show_social_auth_config(self): from djangofloor.management.commands.social_authentications import ( Command as SACommand, ) # noinspection PyCallByClass SACommand.show_config(self)
class Command(BaseCommand): help = ( "show the current configuration." 'Can display as python file ("config python") or as .ini file ("config ini"). Use -v 2 to display more info.' ) requires_system_checks = False options = { "python": "display the current config as Python module", "ini": "display the current config as .ini file", "heroku": "display a configuration valid to deploy on Heroku", "apache": "display an example of Apache config", "nginx": "display an example of Nginx config", "systemd": "display an example of systemd config", "supervisor": "display an example of Supervisor config", "social_authentications": "display configured social authentications", } if settings.USE_CELERY: options["signals"] = "show the defined signals and remote functions" def add_arguments(self, parser): assert isinstance(parser, ArgumentParser) parser.add_argument( "action", default="show", choices=self.options, help=",\n".join(['"%s": %s' % x for x in self.options.items()]), ) parser.add_argument("--filename", default=None, help="write output to this file") remove_arguments_from_help( parser, {"--settings", "--traceback", "--pythonpath"}) def handle(self, *args, **options): try: self.handle_head(**options) except BrokenPipeError: pass def handle_head(self, **options): action = options["action"] verbosity = options["verbosity"] filename = options["filename"] if filename: self.stdout = OutputWrapper(open(filename, "w")) self.style = no_style() if action == "python": self.show_python_config(verbosity) elif action == "ini": self.show_ini_config(verbosity) elif action == "signals": self.show_signals_config() elif action == "heroku": self.show_heroku_config() elif action == "apache": self.show_external_config("djangofloor/config/apache.conf") elif action == "nginx": self.show_external_config("djangofloor/config/nginx.conf") elif action == "systemd": self.show_external_config("djangofloor/config/systemd.conf") elif action == "supervisor": self.show_external_config("djangofloor/config/supervisor.conf") elif action == "social_authentications": self.show_social_auth_config() def show_external_config(self, config): content = render_to_string(config, merger.settings) self.stdout.write(content) def show_signals_config(self): import_signals_and_functions() def display_callable(conn): fn = conn.function if getattr(fn, "__module__", None) and getattr( fn, "__name__", None): path = "%s.%s" % (fn.__module__, fn.__name__) elif getattr(fn, "__name__", None): path = fn.__name__ else: path = str(fn) return path self.stdout.write(self.style.ERROR("Signals")) data = list(decorators.REGISTERED_SIGNALS.items()) for name, connections in sorted(data, key=lambda x: x[0]): self.stdout.write(self.style.WARNING(' "%s"' % name)) for connection in connections: self.stdout.write( self.style.NOTICE(" -> %s" % display_callable(connection))) self.stdout.write(self.style.ERROR("Functions")) data = list(decorators.REGISTERED_FUNCTIONS.items()) for name, connection in sorted(data, key=lambda x: x[0]): self.stdout.write( self.style.WARNING(' "%s" -> %s' % (name, display_callable(connection)))) def show_ini_config(self, verbosity): if verbosity >= 2: self.stdout.write( self.style.SUCCESS("# read configuration files:")) for provider in merger.providers: if not isinstance(provider, IniConfigProvider): continue elif provider.is_valid(): self.stdout.write( self.style.SUCCESS(' # - %s "%s"' % (provider.name, provider))) elif verbosity >= 2: self.stdout.write( self.style.ERROR(' # - %s "%s" (not found)' % (provider.name, provider))) provider = IniConfigProvider() merger.write_provider(provider, include_doc=verbosity >= 2) self.stdout.write(provider.to_str()) def show_python_config(self, verbosity): self.stdout.write(self.style.SUCCESS("# " + "-" * 80)) self.stdout.write( self.style.SUCCESS( _("# Djangofloor version %(version)s") % {"version": version})) self.stdout.write( self.style.SUCCESS( _("# %(project)s version %(version)s") % { "version": guess_version(merger.settings), "project": merger.settings["DF_PROJECT_NAME"], })) self.stdout.write(self.style.SUCCESS("# Configuration providers:")) for provider in merger.providers: if provider.is_valid(): self.stdout.write( self.style.SUCCESS('# - %s "%s"' % (provider.name, provider))) elif verbosity > 1: self.stdout.write( self.style.ERROR('# - %s "%s" (not found)' % (provider.name, provider))) self.stdout.write(self.style.SUCCESS("# " + "-" * 80)) setting_names = list(merger.raw_settings) setting_names.sort() # first, compute all imports to do imports = {} def add_import(val): if not isinstance(val, type): val = val.__class__ if val.__module__ != "builtins": imports.setdefault(val.__module__, set()).add(val.__name__) for setting_name in setting_names: if setting_name not in merger.settings: continue value = merger.settings[setting_name] add_import(value) if imports: self.stdout.write("\n") for module_name in sorted(imports): objects = ", ".join(sorted(imports[module_name])) self.stdout.write( self.style.WARNING("from %s import %s" % (module_name, objects))) self.stdout.write("\n") for setting_name in setting_names: if setting_name not in merger.settings: continue value = merger.settings[setting_name] self.stdout.write( self.style.SUCCESS("%s = %r" % (setting_name, value))) if verbosity <= 1: continue for provider_name, raw_value in merger.raw_settings[ setting_name].items(): self.stdout.write( self.style.WARNING( " # %s -> %r" % (provider_name or "built-in", raw_value))) def show_heroku_config(self): # Pipfile # add extra packages (due to the config) to the Pipfile # requirements.txt # heroku addons:create heroku-postgresql:dev queues = get_expected_queues() self.stdout.write("web: %s-aiohttp" % settings.DF_MODULE_NAME) for queue in queues: self.stdout.write( "%s: %s-%s worker -Q %s" % (queue, settings.DF_MODULE_NAME, "celery", queue)) def show_social_auth_config(self): from djangofloor.management.commands.social_authentications import ( Command as SACommand, ) # noinspection PyCallByClass SACommand.show_config(self)
class Command(BaseCommand): help = ( "show the current configuration." 'Can display as python file ("config python") or as .ini file ("config ini"). Use -v 2 to display more info.' ) requires_system_checks = False options = { "python": "display the current config as Python module", "ini": "display the current config as .ini file", "env": "display the current config as environment variables", } def add_arguments(self, parser: ArgumentParser): parser.add_argument( "action", default="show", choices=self.options, help=",\n".join(['"%s": %s' % x for x in self.options.items()]), ) parser.add_argument("--filename", default=None, help="write output to this file") remove_arguments_from_help( parser, {"--settings", "--traceback", "--pythonpath"}) def handle(self, *args, **options): try: self.handle_head(**options) except BrokenPipeError: pass def handle_head(self, **options): action = options["action"] verbosity = options["verbosity"] filename = options["filename"] fd = None if filename: fd = io.StringIO() self.stdout = OutputWrapper(fd) self.style = no_style() if action == "python": self.show_python_config(verbosity) elif action == "ini": self.show_ini_config(verbosity) elif action == "env": self.show_env_config(verbosity) if filename and action in {"python", "env"}: content = fd.getvalue() # noinspection PyBroadException if action == "python": try: # noinspection PyPackageRequirements,PyUnresolvedReferences import black mode = black.FileMode() # noinspection PyArgumentList content = black.format_file_contents(content, fast=False, mode=mode) except Exception: pass with open(filename, "w") as dst_fd: dst_fd.write(content) def show_external_config(self, config): content = render_to_string(config, merger.settings) self.stdout.write(content) def show_ini_config(self, verbosity): if verbosity >= 2: self.stdout.write( self.style.SUCCESS("# read configuration files:")) for provider in merger.providers: if not isinstance(provider, IniConfigProvider): continue elif provider.is_valid(): self.stdout.write( self.style.SUCCESS(' # - %s "%s"' % (provider.name, provider))) elif verbosity >= 2: self.stdout.write( self.style.ERROR(' # - %s "%s" (not found)' % (provider.name, provider))) provider = IniConfigProvider() merger.write_provider(provider, include_doc=verbosity >= 2) self.stdout.write(provider.to_str()) def show_env_config(self, verbosity): prefix = None for provider in merger.providers: if not isinstance(provider, EnvironmentConfigProvider): continue prefix = provider.prefix if not prefix: self.stderr.write("Environment variables are not used•") return if verbosity >= 2: self.stdout.write( self.style.SUCCESS("# read environment variables:")) provider = EnvironmentConfigProvider(prefix) merger.write_provider(provider, include_doc=verbosity >= 2) self.stdout.write(provider.to_str()) def show_python_config(self, verbosity): self.stdout.write(self.style.SUCCESS("# " + "-" * 80)) self.stdout.write( self.style.SUCCESS( _("# df_config version %(version)s") % {"version": version})) self.stdout.write( self.style.SUCCESS( _("# %(project)s version %(version)s") % { "version": guess_version(merger.settings), "project": merger.settings["DF_PROJECT_NAME"], })) self.stdout.write(self.style.SUCCESS("# Configuration providers:")) for provider in merger.providers: if provider.is_valid(): self.stdout.write( self.style.SUCCESS('# - %s "%s"' % (provider.name, provider))) elif verbosity > 1: self.stdout.write( self.style.ERROR('# - %s "%s" (not found)' % (provider.name, provider))) self.stdout.write(self.style.SUCCESS("# " + "-" * 80)) setting_names = list(merger.raw_settings) setting_names.sort() # first, compute all imports to do imports = {} def add_import(val): if not isinstance(val, type): val = val.__class__ if val.__module__ != "builtins": imports.setdefault(val.__module__, set()).add(val.__name__) for setting_name in setting_names: if setting_name not in merger.settings: continue value = merger.settings[setting_name] add_import(value) if imports: self.stdout.write("\n") for module_name in sorted(imports): objects = ", ".join(sorted(imports[module_name])) self.stdout.write( self.style.WARNING("from %s import %s" % (module_name, objects))) self.stdout.write("\n") for setting_name in setting_names: if setting_name not in merger.settings: continue value = merger.settings[setting_name] self.stdout.write( self.style.SUCCESS("%s = %r" % (setting_name, value))) if verbosity <= 1: continue for provider_name, raw_value in merger.raw_settings[ setting_name].items(): self.stdout.write( self.style.WARNING( " # %s -> %r" % (provider_name or "built-in", raw_value)))
class LoggingMixin: verbosity = 3 if settings.DEBUG else 1 """@type: int""" indent = 0 """@type: int""" logging_level = logging.DEBUG if settings.DEBUG else 1 log_map = dict() logging_initialized = False print_level = True def set_verbosity(self, verbosity): LEVELS = { 0: logging.CRITICAL, 1: logging.ERROR, 2: logging.WARNING, 3: logging.DEBUG, } self.verbosity = verbosity self.logging_level = LEVELS[verbosity] def initialize_logging(self): if not self.logging_initialized: try: self.stdout = OutputWrapper(self.stdout._out, ending='') except AttributeError: self.stdout = OutputWrapper(sys.stdout, ending='') # self.stdout = codecs.getwriter('utf8')(self.stdout) self.logging_initialized = True def color_format(self, level, message): level_colors = { # Level and a pair of colors: first for the label, # the rest for the text; # the bolder color label can make them easier to spot # in the console log. logging.DEBUG: (33, 39), # logging.TRACE: (147, 153), logging.INFO: (43, 49), logging.WARNING: (214, 226), logging.ERROR: (196, 197), logging.CRITICAL: (196, 197), }.get(level, (33, 39)) # 256-color to give wider spectrum than just ANSI color = "\033[38;5;{:d}m" reset = "\033[0m" # Pass any simple messages from internal things, like Django's # runserver, without special formatting. mp_levels = { logging.INFO: u'INF', logging.WARNING: u'WRN', logging.ERROR: u'ERR', logging.DEBUG: u'DBG', logging.CRITICAL: u'CRT' } st_level = mp_levels[level] level_prefix = '%s[%s] ' % (color.format(level_colors[0]), st_level) return u'{level_prefix}{color_normal}{message}{reset}'.format( level_prefix=level_prefix if self.print_level else '', message=message, color_normal=color.format(level_colors[1]), reset=reset ) def llog(self, logging_level, format_string, *args): """ @param logging_level: 50 = summary/critical 40 = error 30 = warning 20 = info 10 = debug @return: """ LEVELS = { logging.CRITICAL, logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG } if logging_level not in LEVELS: logging_level = logging.DEBUG message = format_string % args if logging_level >= self.logging_level: if hasattr(self, 'stdout'): self.initialize_logging() self.stdout.write(u' ' * self.indent) if self.stdout.isatty(): self.stdout.write(self.color_format( logging_level, message)) else: self.stdout.write(message) self.stdout.write('\n') self.log_map.setdefault(logging_level, []).append(message) def log(self, format_string, *args): message = format_string % args if hasattr(self, 'stdout'): self.initialize_logging() self.stdout.write(u' ' * self.indent) self.stdout.write(message) self.stdout.write('\n') def critical(self, format_string, *args): self.llog(logging.CRITICAL, format_string, *args) def debug(self, format_string, *args): self.llog(logging.DEBUG, format_string, *args) def info(self, format_string, *args): self.llog(logging.INFO, format_string, *args) def warning(self, format_string, *args): self.llog(logging.WARNING, format_string, *args) def error(self, format_string, *args): self.llog(logging.ERROR, format_string, *args) def exception(self, format_string, *args): p_type, p_exception, _ = sys.exc_info() self.llog(logging.ERROR, format_string, *args) self.llog(logging.ERROR, u'Exception message: %s', p_exception) self.llog(logging.ERROR, u'Exception type : %s', p_type) self.llog(logging.ERROR, u'Traceback\n%s', format_exc())
class BaseCommand: """ The base class from which all management commands ultimately derive. Use this class if you want access to all of the mechanisms which parse the command-line arguments and work out what code to call in response; if you don't need to change any of that behavior, consider using one of the subclasses defined in this file. If you are interested in overriding/customizing various aspects of the command-parsing and -execution behavior, the normal flow works as follows: 1. ``django-admin`` or ``manage.py`` loads the command class and calls its ``run_from_argv()`` method. 2. The ``run_from_argv()`` method calls ``create_parser()`` to get an ``ArgumentParser`` for the arguments, parses them, performs any environment changes requested by options like ``pythonpath``, and then calls the ``execute()`` method, passing the parsed arguments. 3. The ``execute()`` method attempts to carry out the command by calling the ``handle()`` method with the parsed arguments; any output produced by ``handle()`` will be printed to standard output and, if the command is intended to produce a block of SQL statements, will be wrapped in ``BEGIN`` and ``COMMIT``. 4. If ``handle()`` or ``execute()`` raised any exception (e.g. ``CommandError``), ``run_from_argv()`` will instead print an error message to ``stderr``. Thus, the ``handle()`` method is typically the starting point for subclasses; many built-in commands and command types either place all of their logic in ``handle()``, or perform some additional parsing work in ``handle()`` and then delegate from it to more specialized methods as needed. Several attributes affect behavior at various steps along the way: ``help`` A short description of the command, which will be printed in help messages. ``output_transaction`` A boolean indicating whether the command outputs SQL statements; if ``True``, the output will automatically be wrapped with ``BEGIN;`` and ``COMMIT;``. Default value is ``False``. ``requires_migrations_checks`` A boolean; if ``True``, the command prints a warning if the set of migrations on disk don't match the migrations in the database. ``requires_system_checks`` A boolean; if ``True``, entire Django project will be checked for errors prior to executing the command. Default value is ``True``. To validate an individual application's models rather than all applications' models, call ``self.check(app_configs)`` from ``handle()``, where ``app_configs`` is the list of application's configuration provided by the app registry. ``stealth_options`` A tuple of any options the command uses which aren't defined by the argument parser. """ # Metadata about this command. help = '' # Configuration shortcuts that alter various logic. _called_from_command_line = False output_transaction = False # Whether to wrap the output in a "BEGIN; COMMIT;" requires_migrations_checks = False requires_system_checks = True # Arguments, common to all commands, which aren't defined by the argument # parser. base_stealth_options = ('skip_checks', 'stderr', 'stdout') # Command-specific options not defined by the argument parser. stealth_options = () def __init__(self, stdout=None, stderr=None, no_color=False): self.stdout = OutputWrapper(stdout or sys.stdout) self.stderr = OutputWrapper(stderr or sys.stderr) if no_color: self.style = no_style() else: self.style = color_style() self.stderr.style_func = self.style.ERROR def get_version(self): """ Return the Django version, which should be correct for all built-in Django commands. User-supplied commands can override this method to return their own version. """ return django.get_version() def create_parser(self, prog_name, subcommand): """ Create and return the ``ArgumentParser`` which will be used to parse the arguments to this command. """ parser = CommandParser( prog='%s %s' % (os.path.basename(prog_name), subcommand), description=self.help or None, formatter_class=DjangoHelpFormatter, missing_args_message=getattr(self, 'missing_args_message', None), called_from_command_line=getattr(self, '_called_from_command_line', None), ) # parser.add_argument('--version', action='version', version=self.get_version()) # parser.add_argument( # '-v', '--verbosity', action='store', dest='verbosity', default=1, # type=int, choices=[0, 1, 2, 3], # help='Verbosity level; 0=minimal output, 1=normal output, 2=verbose output, 3=very verbose output', # ) parser.add_argument( '--settings', help=('The Python path to a settings module, e.g. ' '"myproject.settings.main". If this isn\'t provided, the ' 'DJANGO_SETTINGS_MODULE environment variable will be used.'), ) parser.add_argument( '--pythonpath', help= 'A directory to add to the Python path, e.g. "/home/djangoprojects/myproject".', ) parser.add_argument('--traceback', action='store_true', help='Raise on CommandError exceptions') parser.add_argument( '--no-color', action='store_true', dest='no_color', help="Don't colorize the command output.", ) self.add_arguments(parser) return parser def add_arguments(self, parser): """ Entry point for subclassed commands to add custom arguments. """ pass def print_help(self, prog_name, subcommand): """ Print the help message for this command, derived from ``self.usage()``. """ parser = self.create_parser(prog_name, subcommand) parser.print_help() def run_from_argv(self, argv): """ Set up any environment changes requested (e.g., Python path and Django settings), then run this command. If the command raises a ``CommandError``, intercept it and print it sensibly to stderr. If the ``--traceback`` option is present or the raised ``Exception`` is not ``CommandError``, raise it. """ self._called_from_command_line = True parser = self.create_parser(argv[0], argv[1]) options = parser.parse_args(argv[2:]) cmd_options = vars(options) # Move positional args out of options to mimic legacy optparse args = cmd_options.pop('args', ()) handle_default_options(options) try: self.execute(*args, **cmd_options) except Exception as e: if options.traceback or not isinstance(e, CommandError): raise # SystemCheckError takes care of its own formatting. if isinstance(e, SystemCheckError): self.stderr.write(str(e), lambda x: x) else: self.stderr.write('%s: %s' % (e.__class__.__name__, e)) sys.exit(1) finally: try: connections.close_all() except ImproperlyConfigured: # Ignore if connections aren't setup at this point (e.g. no # configured settings). pass def execute(self, *args, **options): """ Try to execute this command, performing system checks if needed (as controlled by the ``requires_system_checks`` attribute, except if force-skipped). """ if options['no_color']: self.style = no_style() self.stderr.style_func = None if options.get('stdout'): self.stdout = OutputWrapper(options['stdout']) if options.get('stderr'): self.stderr = OutputWrapper(options['stderr'], self.stderr.style_func) if self.requires_system_checks and not options.get('skip_checks'): self.check() if self.requires_migrations_checks: self.check_migrations() output = self.handle(*args, **options) if output: if self.output_transaction: connection = connections[options.get('database', DEFAULT_DB_ALIAS)] output = '%s\n%s\n%s' % ( self.style.SQL_KEYWORD( connection.ops.start_transaction_sql()), output, self.style.SQL_KEYWORD( connection.ops.end_transaction_sql()), ) self.stdout.write(output) return output def _run_checks(self, **kwargs): return checks.run_checks(**kwargs) def check(self, app_configs=None, tags=None, display_num_errors=False, include_deployment_checks=False, fail_level=checks.ERROR): """ Use the system check framework to validate entire Django project. Raise CommandError for any serious message (error or critical errors). If there are only light messages (like warnings), print them to stderr and don't raise an exception. """ all_issues = self._run_checks( app_configs=app_configs, tags=tags, include_deployment_checks=include_deployment_checks, ) header, body, footer = "", "", "" visible_issue_count = 0 # excludes silenced warnings if all_issues: debugs = [ e for e in all_issues if e.level < checks.INFO and not e.is_silenced() ] infos = [ e for e in all_issues if checks.INFO <= e.level < checks.WARNING and not e.is_silenced() ] warnings = [ e for e in all_issues if checks.WARNING <= e.level < checks.ERROR and not e.is_silenced() ] errors = [ e for e in all_issues if checks.ERROR <= e.level < checks.CRITICAL and not e.is_silenced() ] criticals = [ e for e in all_issues if checks.CRITICAL <= e.level and not e.is_silenced() ] sorted_issues = [ (criticals, 'CRITICALS'), (errors, 'ERRORS'), (warnings, 'WARNINGS'), (infos, 'INFOS'), (debugs, 'DEBUGS'), ] for issues, group_name in sorted_issues: if issues: visible_issue_count += len(issues) formatted = (self.style.ERROR(str(e)) if e.is_serious() else self.style.WARNING(str(e)) for e in issues) formatted = "\n".join(sorted(formatted)) body += '\n%s:\n%s\n' % (group_name, formatted) if visible_issue_count: header = "System check identified some issues:\n" if display_num_errors: if visible_issue_count: footer += '\n' footer += "System check identified %s (%s silenced)." % ( "no issues" if visible_issue_count == 0 else "1 issue" if visible_issue_count == 1 else "%s issues" % visible_issue_count, len(all_issues) - visible_issue_count, ) if any( e.is_serious(fail_level) and not e.is_silenced() for e in all_issues): msg = self.style.ERROR( "SystemCheckError: %s" % header) + body + footer raise SystemCheckError(msg) else: msg = header + body + footer if msg: if visible_issue_count: self.stderr.write(msg, lambda x: x) else: self.stdout.write(msg) def handle(self, *args, **options): """ The actual logic of the command. Subclasses must implement this method. """ raise NotImplementedError( 'subclasses of BaseCommand must provide a handle() method')
def sync_with_rmis(self, out: OutputWrapper = None, c=None): if out: out.write("Обновление данных для: %s" % self.fio(full=True)) if c is None: from rmis_integration.client import Client c = Client() ok = False has_rmis = False rmis_uid = "" if Card.objects.filter(individual=self, base__is_rmis=True).exists(): rmis_uid = Card.objects.filter(individual=self, base__is_rmis=True)[0].number ok = has_rmis = True if out: out.write("Есть РМИС запись: %s" % rmis_uid) if not ok: docs = Document.objects.filter(individual=self).exclude(document_type__check_priority=0).order_by( "-document_type__check_priority") for document in docs: s = c.patients.search_by_document(document) if len(s) > 0: rmis_uid = s[0] ok = True if out: out.write("Физ.лицо найдено по документу: %s -> %s" % (document, rmis_uid)) break if ok: data = c.patients.get_data(rmis_uid) upd = self.family != data["family"] or self.name != data["name"] or self.patronymic != data[ "patronymic"] or (self.birthday != data["birthday"] and data["birthday"] is not None) if upd: prev = str(self) self.family = data["family"] self.name = data["name"] self.patronymic = data["patronymic"] if data["birthday"] is not None: self.birthday = data["birthday"] self.sex = data["sex"] self.save() if out: out.write("Обновление данных: %s" % self.fio(full=True)) slog.Log(key=str(self.pk), type=2003, body=simplejson.dumps({"Новые данные": str(self), "Не актуальные данные": prev}), user=None).save() if not ok: query = {"surname": self.family, "name": self.name, "patrName": self.patronymic, "birthDate": self.birthday.strftime("%Y-%m-%d")} rows = c.patients.client.searchIndividual(**query) if len(rows) == 1: rmis_uid = rows[0] ok = True if out: out.write("Физ.лицо найдено по ФИО и д.р.: %s" % rmis_uid) if not has_rmis and rmis_uid and rmis_uid != '': ex = Card.objects.filter(number=rmis_uid, is_archive=False, base__is_rmis=True) if ex.exists(): for e in ex: self.join_individual(e.individual, out) s = str(c.patients.create_rmis_card(self, rmis_uid)) if out: out.write("Добавление РМИС карты -> %s" % s) save_docs = [] if ok and rmis_uid != "" and Card.objects.filter(individual=self, base__is_rmis=True, is_archive=False).exists(): pat_data = c.patients.extended_data(rmis_uid) cards = Card.objects.filter(individual=self, base__is_rmis=True, is_archive=False) for card_i in cards: c.patients.sync_card_data(card_i, out) def get_key(d: dict, val): r = [key for key, v in d.items() if v == val] if len(r) > 0: return r[0] return None if out: out.write("Типы документов: %s" % simplejson.dumps(c.patients.local_types)) for document_object in pat_data["identifiers"] or []: k = get_key(c.patients.local_types, document_object["type"]) if k and document_object["active"]: if out: out.write("Тип: %s -> %s (%s)" % (document_object["type"], k, document_object["active"])) data = dict(document_type=DocumentType.objects.get(pk=k), serial=document_object["series"] or "", number=document_object["number"] or "", date_start=document_object["issueDate"], date_end=document_object["expiryDate"], who_give=(document_object["issueOrganization"] or {"name": ""})["name"] or "", individual=self, is_active=True) rowss = Document.objects.filter(document_type=data['document_type'], individual=self, from_rmis=True) if rowss.exclude(serial=data["serial"]).exclude(number=data["number"]).filter( card__isnull=True).exists(): Document.objects.filter(document_type=data['document_type'], individual=self, from_rmis=True).delete() docs = Document.objects.filter(document_type=data['document_type'], serial=data['serial'], number=data['number'], from_rmis=True) if not docs.exists(): doc = Document(**data) doc.save() if out: out.write("Добавление докумена: %s" % doc) kk = "%s_%s_%s" % (doc.document_type.pk, doc.serial, doc.number) save_docs.append(kk) continue else: to_delete = [] has = [] ndocs = {} for d in docs: kk = "%s_%s_%s" % (d.document_type.pk, d.serial, d.number) if out: out.write("Checking: %s" % kk) if kk in has: if out: out.write("to delete: %s" % d.pk) to_delete.append(d.pk) if Card.objects.filter(polis=d).exists(): for c in Card.objects.filter(polis=d): c.polis = ndocs[kk] c.save() else: if out: out.write("To has: %s" % d.pk) has.append(kk) save_docs.append(kk) ndocs[kk] = d Document.objects.filter(pk__in=to_delete).delete() docs = Document.objects.filter(document_type=data['document_type'], serial=data['serial'], number=data['number'], individual=self) for d in docs: if d.date_start != data["date_start"]: d.date_start = data["date_start"] d.save() if out: out.write("Update date_start: %s" % d.date_start) if d.date_end != data["date_end"]: d.date_end = data["date_end"] d.save() if out: out.write("Update date_end: %s" % d.date_end) if d.who_give != data["who_give"]: d.who_give = data["who_give"] d.save() if out: out.write("Update who_give: %s" % d.who_give) if out: out.write("Данные для документов верны: %s" % [str(x) for x in docs]) docs = Document.objects.filter(document_type=data['document_type'], document_type__title__in=['СНИЛС', 'Паспорт гражданина РФ', 'Полис ОМС'], serial=data['serial'], number=data['number']).exclude(individual=self).exclude(number="") if docs.exists(): if out: out.write("Объединение записей физ.лиц") for doc in docs: self.join_individual(doc.individual, out) to_delete_pks = [] for d in Document.objects.filter(individual=self, from_rmis=True): kk = "%s_%s_%s" % (d.document_type.pk, d.serial, d.number) if out: out.write("TD %s %s %s" % (kk, kk not in save_docs, save_docs,)) if kk not in save_docs: to_delete_pks.append(d.pk) Document.objects.filter(pk__in=to_delete_pks).delete() else: if out: out.write("Физ.лицо не найдено в РМИС") return ok
class Command(ProgressBarMixin, BaseCommand): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Country names mapping self.country_names = dict(Country.objects.values_list('name', 'pk')) # Partner entity id mapping self.entity_ids = dict( Partner.objects.annotate(entity_id=Subquery( EntityVersion.objects.filter( entity__organization=OuterRef('organization_id'), parent__isnull=True, ).order_by('-start_date').values('entity_id')[:1]), ).values_list('pk', 'entity_id')) # Existing partner addresses self.partners = self.get_existing_partner_addresses() # Geocoding url self.url = "{esb_api}/{endpoint}".format( esb_api=settings.ESB_API_URL, endpoint=settings.ESB_GEOCODING_ENDPOINT, ) self.counts = defaultdict(int) self.delayed_io = io.StringIO() self.stdout_wrapper = OutputWrapper(self.delayed_io) def add_arguments(self, parser: argparse.ArgumentParser): parser.add_argument( 'csv_file', type=argparse.FileType('r', encoding="utf-8-sig"), ) @staticmethod def check_file(reader): expected_fields = (ID, ERASMUS, STREET_NUM, STREET, POSTAL_CODE, CITY, COUNTRY) try: assert all(field in reader.fieldnames for field in expected_fields) except AssertionError: raise CommandError('Encoding must be UTF-8\n' 'Delimiter must be ";"\n' 'Column names must contain {}\n' 'Headers detected : {}'.format( ", ".join(expected_fields), reader.fieldnames)) @staticmethod def get_existing_partner_addresses(): end_date_condition = (Q(entity_version__end_date__isnull=True) | Q(entity_version__end_date__gte=date.today())) queryset = EntityVersionAddress.objects.filter( end_date_condition, entity_version__entity__organization__partner__isnull=False, ).annotate( partner_id=F('entity_version__entity__organization__partner__id'), ).select_related( 'country', 'entity_version').order_by('-entity_version__start_date') partners = {} for record in queryset: if record.partner_id not in partners: partners[record.partner_id] = record return partners def handle(self, *args, **options): file: io.TextIOWrapper = options['csv_file'] # Read number of lines total = sum(1 for _ in file) - 1 file.seek(0) reader = csv.DictReader(file, delimiter=';') self.check_file(reader) for i, row in enumerate(reader, start=1): try: self.import_address(row, options) except AttributeError: self.stdout_wrapper.write( self.style.ERROR( 'Conflicting entity version for partner {:>4} (entity {})' .format( row[ID], self.entity_ids[int(row[ID])], ))) self.counts['skipped'] += 1 self.print_progress_bar(i, total) self.counts['updated'] += 1 self.print_progress_bar(total, total) self.delayed_io.seek(0) self.stdout.write(self.delayed_io.read()) self.stdout.write( self.style.SUCCESS( 'Updated: {}\nNot found: {}\nSkipped: {}\nWarnings: {}\nExisting: {}' .format( self.counts['updated'], self.counts['not_found'], self.counts['skipped'], self.counts['warning'], self.counts['existing'], ))) @transaction.atomic def import_address(self, row, options): # We need at least the street to update if not row[STREET]: if options['verbosity'] >= 2: self.stdout_wrapper.write( self.style.WARNING( 'Skipping partner id {}: no address'.format(row[ID]))) return # Country name must exist if row[COUNTRY] not in self.country_names: self.counts['skipped'] += 1 self.stdout_wrapper.write( self.style.ERROR( 'Skipping partner id {}: country {} does not exist'.format( row[ID], row[COUNTRY], ))) return # Check that the address has changed existing_address = (self.partners[int(row[ID])] if int(row[ID]) in self.partners else None) if self._is_address_unchanged(row, existing_address): if options['verbosity'] >= 2: self.stdout_wrapper.write( self.style.WARNING( 'Skipping partner id {}: same address'.format( row[ID]))) self.counts['existing'] += 1 return parts = [ row[STREET_NUM], row[STREET], row[POSTAL_CODE], row[CITY], row[COUNTRY], ] search = ' '.join(filter(None, parts)) response = requests.get(self.url, {'address': search}, headers={ 'Authorization': settings.ESB_AUTHORIZATION, }) results = response.json()['results'] if not results: self.counts['not_found'] += 1 self.stdout_wrapper.write( self.style.ERROR( 'Address not found for partner id {:>4} not found: {}'. format( row[ID], search, ))) return if len(results) > 1: self.counts['warning'] += 1 self.stdout_wrapper.write( self.style.WARNING('Multiple results for partner id {}'.format( row[ID]))) location = results[0]['geometry']['location'] if options['verbosity'] >= 2: self.stdout_wrapper.write( self.style.SUCCESS( 'Address found for partner id {:>4} : {}, {}'.format( row[ID], location['lat'], location['lng'], ))) # Handle entity version last_version = None if existing_address: last_version = self._override_current_version(existing_address) if not last_version: entity_id = self.entity_ids[int(row[ID])] entity = Entity.objects.select_related('organization').get( pk=entity_id) # Create a new entity version last_version = EntityVersion.objects.create( title=entity.organization.name, entity_id=entity_id, parent=None, start_date=date.today(), end_date=None, ) # Create the address EntityVersionAddress.objects.create( street_number=row[STREET_NUM], street=row[STREET], postal_code=row[POSTAL_CODE], city=row[CITY], location=Point( location['lng'], location['lat'], ), country_id=self.country_names[row[COUNTRY]], entity_version=last_version, ) @staticmethod def _override_current_version(existing_address): """ Update a previous version of an address, or override an existing one :param existing_address: existing EntityVersionAddress :return: EntityVersion or None if a new vesion must be created """ today = date.today() last_version = existing_address.entity_version if last_version.start_date != today: # End the previous version if start_date is changed last_version.end_date = today - timedelta(days=1) last_version.save() # We can safely create a new version last_version = None else: # Latest version date is today, delete the related address (if existing) last_version.entityversionaddress_set.all().delete() return last_version @staticmethod def _is_address_unchanged(row, address): """ Check if there are differences between imported address and existing :param row: Dict address from CSV file :param address: existing EntityVersionAddress or None :return: True if the two address are the same """ return address and all([ row[STREET_NUM] == address.street_number, row[STREET] == address.street, row[POSTAL_CODE] == address.postal_code, row[CITY] == address.city, row[COUNTRY] == (address.country and address.country.name), ])
class BaseCommand(DjBaseCommand): """ A less terrible base class for Djagno management commands. Changes: * Tracebacks are turned on by default * Verbosity is controlled by a counter instead of a number (ex, ``-vvv`` instead of ``-v3``) and are used to set up logging levels too (disable by setting ``logging_enabled = False``). * Will override a named logger (see ``logging_handler_name``) with the level and formatter settings from the command line (or add a new stderr logger if no matching logger is found). * Subclassess can set ``option_list`` directly - the base options are stored in ``base_option_list`` and merged with ``get_option_list``. * The ``self.log`` attribute will be set to a logger configured to use the name from ``self.get_log_name()`` (which defaults to the fully qualified name of the module containing the command). * If ``handle`` returns an integer it will be treated as a numeric exit status. If a ``str`` or ``unicode`` is returned it will be treated "normally" and 0 will be returned. * If ``handle`` raises an exception that exception will be logged (unless ``self.log_exc`` is ``False``) and the command will exit with a status of 1. Example:: from dwdj.management.base import BaseCommand, make_option class MyCommand(BaseCommand): option_list = [ make_option("-f", "--foo", action="store_true"), ] def handle(self, *args, **options): if options.get("foo"): return 1 return 0 """ # For now, tell Django to use the legacy optparse option parsing. At some # point we'll need to update all the option parsing so it's done the fancy # new way with argparse, but that day is not today. use_argparse = False base_option_list = [ make_option('-q', '--quiet', action="store_const", const=-1, dest="verbosity"), make_option('-v', '--verbose', action="count", default=0, dest="verbosity"), make_option('--verbose-log', action="store_true", help=dedent(""" Use a more verbose logging format. """)), make_option('--settings', help=dedent(""" The Python path to a settings module, e.g. "myproject.settings.main". If this isn\'t provided, the DJANGO_SETTINGS_MODULE environment variable will be used.') """)), make_option('--pythonpath', help=dedent(""" A directory to add to the Python path, e.g. "/home/djangoprojects/myproject". """)), make_option('--traceback', action='store_true', default=True, help=dedent(""" Log complete exception traceback, not just exception and message. """)), make_option('--no-traceback', action='store_false', dest="traceback", help=dedent(""" Log only exception messages, not complete tracebacks. """)) ] option_list = [] logging_enabled = True logging_handler_name = "stderr" logging_format = '%(levelname)s [%(name)s]: %(message)s' logging_format_verbose = '%(asctime)s %(levelname)s [%(processName)s:%(threadName)s:%(name)s]: %(message)s' log_exc = True def get_option_list(self): return self.base_option_list + self.option_list def create_parser(self, prog_name, subcommand): """ Create and return the ``OptionParser`` which will be used to parse the arguments to this command. """ parser = OptionParser(prog=prog_name, usage=self.usage(subcommand), version=self.get_version(), option_list=self.get_option_list()) original_parse = parser.parse_args parser.parse_args = partial(self._override_parse_args, parser, original_parse) return parser def _override_parse_args(self, parser, original_parse, *argv, **kwargs): options, args = original_parse(*argv, **kwargs) self.logging_setup(options) if options.verbosity < 0: options.traceback = False return (options, args) def logging_setup(self, options): verbosity = options.verbosity level = ( logging.CRITICAL if verbosity < 0 else logging.ERROR if verbosity == 0 else logging.WARNING if verbosity == 1 else logging.INFO if verbosity == 2 else logging.DEBUG ) format = ( self.logging_format if not options.verbose_log else self.logging_format_verbose ) logger, handler = self.logging_get_handler() formatter = logging.Formatter(format) handler.setFormatter(formatter) if logger.level > level: logger.setLevel(level) handler.setLevel(level) def get_log_name(self): return type(self).__module__ @cached_property def log(self): return logging.getLogger(self.get_log_name()) def logging_get_handler(self): logger = logging.getLogger("") for handler in logger.handlers: if handler.name == self.logging_handler_name: return logger, handler handler = logging.StreamHandler(sys.stderr) handler.name = self.logging_handler_name logger.addHandler(handler) return logger, handler def run_from_argv(self, argv): """ Set up any environment changes requested (e.g., Python path and Django settings), then run this command. If the command raises a ``CommandError``, intercept it and print it sensibly to stderr. """ signal_pre_run.send(self, argv=argv) parser = self.create_parser(argv[0], argv[1]) options, args = parser.parse_args(argv[2:]) handle_default_options(options) try: result = self.execute(*args, **options.__dict__) except SystemExit as e: signal_post_run.send(self, status=e.code) raise except BaseException: result = self.handle_execute_exc(options) if result is None: result = 1 status = result or 0 signal_post_run.send(self, status=status) sys.exit(status) def handle_execute_exc(self, options): if not self.log_exc: return 1 if options.traceback: self.log.exception("Exception running command:") else: exc_info = sys.exc_info() self.log.error("%s: %s", exc_info[0].__name__, exc_info[1]) return 1 def execute(self, *args, **options): """ Try to execute this command, performing model validation if needed (as controlled by the attribute ``self.requires_model_validation``, except if force-skipped). """ # Switch to English, because django-admin.py creates database content # like permissions, and those shouldn't contain any translations. # But only do this if we can assume we have a working settings file, # because django.utils.translation requires settings. saved_lang = None self.stdout = OutputWrapper(options.get('stdout', sys.stdout)) self.stderr = OutputWrapper(options.get('stderr', sys.stderr), self.style.ERROR) if self.can_import_settings: from django.utils import translation saved_lang = translation.get_language() translation.activate('en-us') try: if getattr(self, "requires_model_validation", False) and not options.get('skip_validation'): self.validate() result = self.handle(*args, **options) if isinstance(result, basestring): if self.output_transaction: # This needs to be imported here, because it relies on # settings. from django.db import connections, DEFAULT_DB_ALIAS connection = connections[options.get('database', DEFAULT_DB_ALIAS)] if connection.ops.start_transaction_sql(): self.stdout.write(self.style.SQL_KEYWORD(connection.ops.start_transaction_sql())) self.stdout.write(result) if self.output_transaction: self.stdout.write('\n' + self.style.SQL_KEYWORD("COMMIT;")) result = 0 return result finally: if saved_lang is not None: translation.activate(saved_lang)
class Daemon(BaseDaemon): def __init__(self, wait=2, threshold=4, stdout=None, stderr=None, no_color=False): """Task Daemon: Gets task and execute them. :arg wait : How many seconds should wait before check new records again. This should be lower than threshold :arg threshold : """ self.wait = wait self.threshold = threshold self.threads = [] self.queues = [] self.stdout = OutputWrapper(stdout or sys.stdout) self.stderr = OutputWrapper(stderr or sys.stderr) if no_color: self.style = no_style() else: self.style = color_style() self.stderr.style_func = self.style.ERROR super(Daemon, self).__init__(name="Daemon", pidfile=_pidfile, runfile=_runfile, stoptimeout=10, debug=1) def start(self): # Check daemon is running pid = self.__getpid() if pid: logger.warning("Daemon: Already Running, can not start.") raise DaemonRunning() # Start Daemon logger.info("Daemon: Started.") self.daemonize() self.run() def stop(self): # Check alive daemon if not exists already pid = self.__getpid() if not pid: logger.warning("Daemon: Not Running, can not stop.") raise DaemonNotRunning() self.stdout.write(self.style.WARNING("Trying to stop...")) self.__cleanstop(pid) try: while True: os.kill(pid, signal.SIGTERM) time.sleep(0.1) except OSError as err: err = str(err) if err.find('No such process') > 0: if os.path.exists(self.pidfile): os.remove(self.pidfile) else: raise DaemonError(err) def is_running(self): """Checks daemon is running""" return os.path.exists(self.runfile) @staticmethod def _is_queue_time_came(q: Queue): return q.timer <= timezone.now() def start_queue(self, q: Queue): if not self._is_queue_time_came(q): return logger.debug("Daemon: Start Queue<%d>" % q.id) try: thread = QueueThread(q.id) thread.start() self.threads.append(thread) except Exception: logger.exception("Daemon: Start Queue<%d> failed." % q.id) try: q.set_status_error() except: logger.exception( "Daemon: Queue<%d> status can not set error." % q.id) def queue_timeout(self, q: Queue): try: logger.warning("Queue<%d> timeout, changing status." % q.id) q.status = QueueStatus.Timeout.value q.save() except Exception: logger.exception("Queue<%d> status can not changed to Timeout." % q.id) @staticmethod def get_queues(stat: QueueStatus): return Queue.objects.all().filter(status=stat.value).order_by('timer') def _add_queue_list(self, q: Queue): if q.id not in self.queues: logger.debug("Daemon: Queue<%d> is adding to the queues." % q.id) self.queues.append(q.id) else: logger.debug("Daemon: Queue<%d> is already in queues." % q.id) def run(self): start_time = timezone.now() try: while self.is_running(): for queue in self.get_queues(QueueStatus.Processing): queue.calculate_queue_status() for queue in self.get_queues(QueueStatus.Created): if queue.timer: # Check is timeout if queue.timer < timezone.now() - timezone.timedelta( seconds=self.threshold): self.queue_timeout(queue) else: self.start_queue(queue) else: self.start_queue(queue) # Log every 10 seconds passed = (timezone.now() - start_time).total_seconds() if int(passed) % 10 == 0: logger.debug("Daemon: Running %d seconds." % passed) time.sleep(self.wait) except Exception: logger.exception("Daemon: Failed.") self.delrun() raise DaemonError() logger.warning("Daemon: Exiting.")
def parse_logs(qs, stdout=None): """ Parse logs for kudos. """ names = collections.deque(maxlen=200) unattributed = 0 count = 0 kudos = {} kudos_count = 0 kudos_first = {} kudos_recent = {} if stdout and not isinstance(stdout, OutputWrapper): stdout = OutputWrapper(stdout) def set_thanked(nick): timestamp = log[3] kudos[nick] = kudos.get(nick, 0) + 1 kudos_first.setdefault(nick, timestamp) kudos_recent[nick] = timestamp qs = qs.order_by('pk').filter(command='PRIVMSG') qs = qs.values_list('pk', 'nick', 'text', 'timestamp') for log in _iterate_log(qs): log_nick = log[1].lower() log_text = log[2] count += 1 directed = directed_message(log_text) if directed: directed = directed.lower() if directed == log_nick: # Can't thank yourself :P directed = None if RE_KUDOS.search(log_text): kudos_count += 1 attributed = False if directed: for nick, _ in names: if nick == directed: set_thanked(nick) attributed = True break if not attributed: lower_text = log_text.lower() for recent in (bits[0] for bits in names if bits[0] != log_nick): re_text = '(?:^| )@?{}(?:$|\W)'.format(re.escape(recent)) if re.search(re_text, lower_text): set_thanked(recent) attributed = True if not attributed: for nick, directed in names: if directed == log_nick: set_thanked(nick) attributed = True break if not attributed: unattributed += 1 names.append((log_nick, directed)) if stdout and not count % 10000: stdout.write('.', ending='') stdout.flush() if stdout: stdout.write('') kudos_list = [] for c, nick in sorted((c, nick) for nick, c in kudos.items()): kudos_list.append({ 'nick': nick, 'count': c, 'first': kudos_first[nick], 'recent': kudos_recent[nick] }) return { 'kudos': kudos_list, 'message_count': count, 'kudos_given': kudos_count, 'unattributed': unattributed, }
class SettingMerger: """Load different settings modules and config files and merge them. """ def __init__( self, fields_provider, providers, extra_values=None, stdout=None, stderr=None, no_color=False, ): self.fields_provider = fields_provider or PythonConfigFieldsProvider(None) extra_values = extra_values or {} self.extra_values = extra_values self.providers = providers or [] self.__formatter = string.Formatter() self.settings = {} self.config_values = ( [] ) # list of (ConfigValue, provider_name, setting_name, final_value) self.raw_settings = OrderedDict() for key, value in extra_values.items(): self.raw_settings[key] = OrderedDict() self.raw_settings[key][None] = value # raw_settings[setting_name][str(provider) or None] = raw_value self.__working_stack = set() self.stdout = OutputWrapper(stdout or sys.stdout) self.stderr = OutputWrapper(stderr or sys.stderr) if no_color: self.style = no_style() else: self.style = color_style() self.stderr.style_func = self.style.ERROR def add_provider(self, provider): self.providers.append(provider) def process(self): self.load_raw_settings() self.load_settings() def load_raw_settings(self): # get all setting names and sort them all_settings_names_set = set() for field in self.fields_provider.get_config_fields(): assert isinstance(field, ConfigField) all_settings_names_set.add(field.setting_name) for provider in self.providers: assert isinstance(provider, ConfigProvider) for setting_name, value in provider.get_extra_settings(): all_settings_names_set.add(setting_name) all_settings_names = list(sorted(all_settings_names_set)) # initialize all defined settings for setting_name in all_settings_names: self.raw_settings[setting_name] = OrderedDict() # fetch default values if its exists (useless?) for field in self.fields_provider.get_config_fields(): assert isinstance(field, ConfigField) self.raw_settings[field.setting_name][None] = field.value # read all providers (in the right order) for provider in self.providers: assert isinstance(provider, ConfigProvider) source_name = str(provider) for field in self.fields_provider.get_config_fields(): assert isinstance(field, ConfigField) if provider.has_value(field): value = provider.get_value(field) # noinspection PyTypeChecker self.raw_settings[field.setting_name][source_name] = value for setting_name, value in provider.get_extra_settings(): self.raw_settings[setting_name][source_name] = value def has_setting_value(self, setting_name): return setting_name in self.raw_settings def get_setting_value(self, setting_name): if setting_name in self.settings: return self.settings[setting_name] elif setting_name in self.__working_stack: raise ValueError( "Invalid cyclic dependency between " + ", ".join(self.__working_stack) ) elif setting_name not in self.raw_settings: raise ValueError("Invalid setting reference: %s" % setting_name) self.__working_stack.add(setting_name) provider_name, raw_value = None, None for provider_name, raw_value in self.raw_settings[setting_name].items(): pass value = self.analyze_raw_value(raw_value, provider_name, setting_name) self.settings[setting_name] = value self.__working_stack.remove(setting_name) return value def load_settings(self): for setting_name in self.raw_settings: self.get_setting_value(setting_name) def call_method_on_config_values(self, method_name: str): """Scan all settings, looking for :class:`django.conf.config_values.ConfigValue` and calling one of their methods. :param method_name: 'pre_collectstatic', 'pre_migrate', 'post_collectstatic', or 'post_migrate'. """ for raw_value, provider_name, setting_name, final_value in self.config_values: try: getattr(raw_value, method_name)( self, provider_name, setting_name, final_value ) except Exception as e: self.stdout.write( self.style.ERROR( 'Invalid value "%s" in %s for %s (%s)' % (raw_value, provider_name or "built-in", setting_name, e) ) ) def analyze_raw_value(self, obj, provider_name, setting_name): """Parse the object for replacing variables by their values. If `obj` is a string like "THIS_IS_{TEXT}", search for a setting named "TEXT" and replace {TEXT} by its value (say, "VALUE"). The returned object is then equal to "THIS_IS_VALUE". If `obj` is a list, a set, a tuple or a dict, its components are recursively parsed. If `obj` is a subclass of :class:`djangofloor.conf.config_values.ConfigValue`, its value is on-the-fly computed. Otherwise, `obj` is returned as-is. :param obj: object to analyze :param provider_name: the name of the config file :param setting_name: the name of the setting containing this value but this value can be inside a dict or a list (like `SETTING = [Directory("/tmp"), ]`) :return: the parsed setting """ if isinstance(obj, str): values = {} for ( literal_text, field_name, format_spec, conversion, ) in self.__formatter.parse(obj): if field_name is not None: values[field_name] = self.get_setting_value(field_name) return self.__formatter.format(obj, **values) elif isinstance(obj, ConfigValue): final_value = obj.get_value(self, provider_name, setting_name) self.config_values.append((obj, provider_name, setting_name, final_value)) return final_value elif isinstance(obj, list) or isinstance(obj, tuple): result = [] for sub_obj in obj: if isinstance(sub_obj, ExpandIterable): result += self.get_setting_value(sub_obj.value) else: result.append( self.analyze_raw_value(sub_obj, provider_name, setting_name) ) if isinstance(obj, tuple): return tuple(result) return result elif isinstance(obj, set): result = set() for sub_obj in obj: if isinstance(sub_obj, ExpandIterable): result |= self.get_setting_value(sub_obj.value) else: result.add( self.analyze_raw_value(sub_obj, provider_name, setting_name) ) return result elif isinstance(obj, dict): result = obj.__class__() # OrderedDict or plain dict for sub_key, sub_obj in obj.items(): if isinstance(sub_obj, ExpandIterable): result.update(self.get_setting_value(sub_obj.value)) else: value = self.analyze_raw_value(sub_obj, provider_name, setting_name) key = self.analyze_raw_value(sub_key, provider_name, setting_name) result[key] = value return result return obj def post_process(self): """Perform some cleaning on settings: * remove duplicates in `INSTALLED_APPS` (keeps only the first occurrence) """ # remove duplicates in INSTALLED_APPS self.settings["INSTALLED_APPS"] = list( OrderedDict.fromkeys(self.settings["INSTALLED_APPS"]) ) django_version = get_version() # remove deprecated settings if LooseVersion(django_version) >= LooseVersion("1.8"): if "TEMPLATES" in self.settings: for key in ( "TEMPLATE_DIRS", "TEMPLATE_CONTEXT_PROCESSORS", "TEMPLATE_LOADERS", "TEMPLATE_DEBUG", ): if key in self.settings: del self.settings[key] def write_provider(self, provider, include_doc=False): for config_field in sorted( self.fields_provider.get_config_fields(), key=lambda x: x.name ): assert isinstance(config_field, ConfigField) if config_field.setting_name not in self.settings: continue config_field.value = self.settings[config_field.setting_name] provider.set_value(config_field, include_doc=include_doc)
def parse_logs(qs, stdout=None): """ Parse logs for kudos. """ names = collections.deque(maxlen=200) unattributed = 0 count = 0 kudos = {} kudos_count = 0 kudos_first = {} kudos_recent = {} if stdout and not isinstance(stdout, OutputWrapper): stdout = OutputWrapper(stdout) def set_thanked(nick): timestamp = log[3] kudos[nick] = kudos.get(nick, 0) + 1 kudos_first.setdefault(nick, timestamp) kudos_recent[nick] = timestamp qs = qs.order_by('pk').filter(command='PRIVMSG') qs = qs.values_list('pk', 'nick', 'text', 'timestamp') for log in _iterate_log(qs): log_nick = log[1].lower() log_text = log[2] count += 1 directed = directed_message(log_text) if directed: directed = directed.lower() if directed == log_nick: # Can't thank yourself :P directed = None if RE_KUDOS.search(log_text): kudos_count += 1 attributed = False if directed: for nick, _ in names: if nick == directed: set_thanked(nick) attributed = True break if not attributed: lower_text = log_text.lower() for recent in ( bits[0] for bits in names if bits[0] != log_nick): re_text = '(?:^| )@?{}(?:$|\W)'.format(re.escape(recent)) if re.search(re_text, lower_text): set_thanked(recent) attributed = True if not attributed: for nick, directed in names: if directed == log_nick: set_thanked(nick) attributed = True break if not attributed: unattributed += 1 names.append((log_nick, directed)) if stdout and not count % 10000: stdout.write('.', ending='') stdout.flush() if stdout: stdout.write('') kudos_list = [] for c, nick in sorted((c, nick) for nick, c in kudos.items()): kudos_list.append({ 'nick': nick, 'count': c, 'first': kudos_first[nick], 'recent': kudos_recent[nick] }) return { 'kudos': kudos_list, 'message_count': count, 'kudos_given': kudos_count, 'unattributed': unattributed, }