Exemplo n.º 1
0
    def handle(self, *args, **options):
        sys.path.append(os.path.abspath('scrapers'))

        # @see https://github.com/opencivicdata/pupa/blob/master/pupa/cli/__main__.py
        parser = argparse.ArgumentParser('pupa')
        subparsers = parser.add_subparsers(dest='subcommand')
        subcommand = importlib.import_module(
            'pupa.cli.commands.update').Command(subparsers)

        logging.config.dictConfig(pupa_settings.LOGGING)
        handler = logging.getLogger().handlers[0]
        module_names = args or os.listdir('scrapers')
        prepend_args = ['update']
        if options['fastmode']:
            prepend_args.append('--fastmode')
        append_args = ['people']

        # @see https://pythonhosted.org//logutils/testing.html
        # @see http://plumberjack.blogspot.ca/2010/09/unit-testing-and-logging.html
        for module_name in module_names:
            if os.path.isdir(os.path.join(
                    'scrapers', module_name)) and module_name not in (
                        '.git', '_cache', '_data', '__pycache__', 'disabled'):
                report, _ = Report.objects.get_or_create(module=module_name)
                try:
                    with transaction.atomic():
                        flush(module_name)
                        known_args = prepend_args[:]
                        known_args.append(module_name)
                        known_args.extend(append_args)
                        args, other = parser.parse_known_args(known_args)
                        report.report = subcommand.handle(args, other)
                        report.exception = ''
                        report.success_at = datetime.now()
                except:
                    report.exception = traceback.format_exc()
                report.warnings = '\n'.join(
                    '%(asctime)s %(levelname)s %(name)s: %(message)s' % d
                    for d in handler.buffer
                    if ' memberships, ' not in d['message'])
                report.save()
                handler.flush()
Exemplo n.º 2
0
    def handle(self, *args, **options):
        sys.path.append(os.path.abspath('scrapers'))

        # @see https://github.com/opencivicdata/pupa/blob/master/pupa/cli/__main__.py
        parser = argparse.ArgumentParser('pupa')
        subparsers = parser.add_subparsers(dest='subcommand')
        subcommand = importlib.import_module('pupa.cli.commands.update').Command(subparsers)

        logging.config.dictConfig(pupa_settings.LOGGING)
        handler = logging.getLogger().handlers[0]
        module_names = args or os.listdir('scrapers')
        prepend_args = ['update']
        if options['fastmode']:
            prepend_args.append('--fastmode')
        append_args = ['people']

        # @see https://pythonhosted.org//logutils/testing.html
        # @see http://plumberjack.blogspot.ca/2010/09/unit-testing-and-logging.html
        for module_name in module_names:
            if os.path.isdir(os.path.join('scrapers', module_name)) and module_name not in ('.git', '_cache', '_data', '__pycache__', 'disabled'):
                report, _ = Report.objects.get_or_create(module=module_name)
                try:
                    with transaction.atomic():
                        flush(module_name)
                        known_args = prepend_args[:]
                        known_args.append(module_name)
                        known_args.extend(append_args)
                        args, other = parser.parse_known_args(known_args)
                        report.report = subcommand.handle(args, other)
                        report.exception = ''
                        report.success_at = datetime.now()
                except:
                    report.exception = traceback.format_exc()
                report.warnings = '\n'.join('%(asctime)s %(levelname)s %(name)s: %(message)s' % d for d in handler.buffer if ' memberships, ' not in d['message'])
                report.save()
                handler.flush()
Exemplo n.º 3
0
    def handle(self, *args, **options):
        sys.path.append(os.path.abspath('scrapers'))

        for module_name in options['module']:
            flush(module_name)