def registry(base='.'): """ Reads definition.py files. """ boundaries.autodiscover(base) return boundaries.registry
def handle(self, *args, **options): # Load configuration boundaries.autodiscover(options['data_dir']) all_sources = boundaries.registry all_slugs = set(slugify(s) for s in all_sources) if options['only']: only = set(options['only'].split(',')) sources = only.intersection(all_slugs) elif options['except']: exceptions = set(options['except'].split(',')) sources = all_slugs - exceptions else: sources = all_slugs for slug, config in all_sources.items(): # Backwards compatibility with specifying the name, rather than the slug, # as the first arg in the definition config.setdefault('name', slug) slug = slugify(slug) if slug not in sources: log.debug('Skipping %s.' % slug) continue if (not options['reload']) and BoundarySet.objects.filter(slug=slug).exists(): log.info('Already loaded %s, skipping.' % slug) continue self.load_set(slug, config, options)
def test_autodiscover(self): boundaries.registry = {} boundaries._basepath = '.' with LogCapture() as l: boundaries.autodiscover('.') self.assertEqual(boundaries.registry, {'Districts': {'file': './boundaries/tests/fixtures/', 'last_updated': date(2000, 1, 1)}}) l.check(('boundaries', 'WARNING', 'Multiple definitions of Districts found.'))
def test_autodiscover(self): # Uses bar_definition.py and foo_definition.py fixtures. boundaries.registry = {} boundaries._basepath = '.' with LogCapture() as l: boundaries.autodiscover('./boundaries/tests/fixtures') self.assertEqual(len(boundaries.registry), 1) self.assertEqual(boundaries.registry['Districts']['file'], './boundaries/tests/fixtures/foo.shp') self.assertEqual(boundaries.registry['Districts']['last_updated'], date(2000, 1, 1)) l.check(('boundaries', 'WARNING', 'Multiple definitions of Districts found.'))
def handle(self, *args, **options): if settings.DEBUG: print( _('DEBUG is True. This can cause memory usage to balloon. Continue? [y/n]' )) if input().lower() != 'y': return boundaries.autodiscover(options['data_dir']) if options['only']: whitelist = set(options['only'].split(',')) else: whitelist = set() if options['except']: blacklist = set(options['except'].split(',')) else: blacklist = set() with alive_bar(len(boundaries.registry.items())) as bar: for slug, definition in boundaries.registry.items(): bar() name = slug slug = slugify(slug) if self.loadable(slug, definition['last_updated'], whitelist, blacklist, options['reload']): #log.info(_('Processing %(slug)s.') % {'slug': slug}) # Backwards-compatibility with having the name, instead of the slug, # as the first argument to `boundaries.register`. definition.setdefault('name', name) definition = Definition(definition) data_sources, tmpdirs = create_data_sources( definition['file'], encoding=definition['encoding'], convert_3d_to_2d=options['clean']) try: if not data_sources: log.warning(_('No shapefiles found.')) else: self.load_boundary_set(slug, definition, data_sources, options) finally: for tmpdir in tmpdirs: rmtree(tmpdir) else: log.debug(_('Skipping %(slug)s.') % {'slug': slug})
def handle(self, *args, **options): if settings.DEBUG: print(_("DEBUG is True. This can cause memory usage to balloon. Continue? [y/n]")) if six.moves.input().lower() != "y": return boundaries.autodiscover(options["data_dir"]) if options["only"]: whitelist = set(options["only"].split(",")) else: whitelist = set() if options["except"]: blacklist = set(options["except"].split(",")) else: blacklist = set() for slug, definition in boundaries.registry.items(): name = slug slug = slugify(slug) if self.loadable(slug, definition["last_updated"], whitelist, blacklist, options["reload"]): log.info(_("Processing %(slug)s.") % {"slug": slug}) # Backwards-compatibility with having the name, instead of the slug, # as the first argument to `boundaries.register`. definition.setdefault("name", name) definition = Definition(definition) data_sources, tmpdirs = create_data_sources( definition["file"], definition["encoding"], options["clean"] ) try: if not data_sources: log.warning(_("No shapefiles found.")) else: self.load_boundary_set(slug, definition, data_sources, options) finally: for tmpdir in tmpdirs: rmtree(tmpdir) else: log.debug(_("Skipping %(slug)s.") % {"slug": slug})
def handle(self, *args, **options): if settings.DEBUG: print('DEBUG is True - this can cause memory usage to balloon. continue? [y/n]') if six.moves.input().lower() != 'y': return # Load configuration boundaries.autodiscover(options['data_dir']) all_sources = boundaries.registry all_slugs = set(slugify(s) for s in all_sources) if options['only']: only = set(options['only'].split(',')) sources = only.intersection(all_slugs) elif options['except']: exceptions = set(options['except'].split(',')) sources = all_slugs - exceptions else: sources = all_slugs for slug, config in all_sources.items(): # Backwards compatibility with specifying the name, rather than the slug, # as the first arg in the definition config.setdefault('name', slug) slug = slugify(slug) if slug not in sources: log.debug('Skipping %s.' % slug) continue try: existing_set = BoundarySet.objects.get(slug=slug) if (not options['reload']) and existing_set.last_updated >= config['last_updated']: log.info('Already loaded %s, skipping.' % slug) continue except BoundarySet.DoesNotExist: pass self.load_set(slug, config, options)
def handle(self, *args, **options): if settings.DEBUG: print(_('DEBUG is True. This can cause memory usage to balloon. Continue? [y/n]')) if six.moves.input().lower() != 'y': return boundaries.autodiscover(options['data_dir']) if options['only']: whitelist = set(options['only'].split(',')) else: whitelist = set() if options['except']: blacklist = set(options['except'].split(',')) else: blacklist = set() for slug, definition in boundaries.registry.items(): name = slug slug = slugify(slug) if self.loadable(slug, definition['last_updated'], whitelist, blacklist, options['reload']): log.info(_('Processing %(slug)s.') % {'slug': slug}) # Backwards-compatibility with having the name, instead of the slug, # as the first argument to `boundaries.register`. definition.setdefault('name', name) definition = Definition(definition) data_sources, tmpdirs = create_data_sources(definition['file'], definition['encoding'], options['clean']) try: if not data_sources: log.warning(_('No shapefiles found.')) else: self.load_boundary_set(slug, definition, data_sources, options) finally: for tmpdir in tmpdirs: rmtree(tmpdir) else: log.debug(_('Skipping %(slug)s.') % {'slug': slug})
def handle(self, *args, **options): boundaries.autodiscover(options['data_dir']) for slug in sorted(boundaries.registry): name = slug slug = slugify(slug) definition = boundaries.registry[name] # Backwards-compatibility with having the name, instead of the slug, # as the first argument to `boundaries.register`. definition.setdefault('name', name) definition = Definition(definition) data_sources, tmpdirs = create_data_sources(definition['file'], encoding=definition['encoding']) try: if not data_sources: log.warning(_('No shapefiles found.')) else: features = OrderedDict() for data_source in data_sources: features[slug] = [] layer = data_source[0] for feature in layer: feature = Feature(feature, definition) if feature.is_valid(): features[slug].append((feature.id, feature.name)) for slug, features in features.items(): print('\n%s: %d' % (slug, len(features))) for properties in sorted(features): print('%s: %s' % properties) finally: for tmpdir in tmpdirs: rmtree(tmpdir)
def handle(self, *args, **options): boundaries.autodiscover(options['data_dir']) for slug, definition in boundaries.registry.items(): name = slug slug = slugify(slug) # Backwards-compatibility with having the name, instead of the slug, # as the first argument to `boundaries.register`. definition.setdefault('name', name) definition = Definition(definition) data_sources, tmpdirs = create_data_sources(definition['file'], encoding=definition['encoding']) try: if not data_sources: log.warning(_('No shapefiles found.')) else: features = OrderedDict() for data_source in data_sources: features[slug] = [] layer = data_source[0] for feature in layer: feature = Feature(feature, definition) if feature.is_valid(): features[slug].append((feature.id, feature.name)) for slug, features in features.items(): print('\n%s: %d' % (slug, len(features))) for properties in sorted(features): print('%s: %s' % properties) finally: for tmpdir in tmpdirs: rmtree(tmpdir)
def registry(base='.'): import boundaries boundaries.autodiscover(base) return boundaries.registry