def registry(base='.'): """ Reads definition.py files. """ boundaries.autodiscover(base) return boundaries.registry
def registry(base='.'):
"""
Reads definition.py files.
"""
boundaries.autodiscover(base)
return boundaries.registry
def handle(self, *args, **options):
# Load configuration
boundaries.autodiscover(options['data_dir'])
all_sources = boundaries.registry
all_slugs = set(slugify(s) for s in all_sources)
if options['only']:
only = set(options['only'].split(','))
sources = only.intersection(all_slugs)
elif options['except']:
exceptions = set(options['except'].split(','))
sources = all_slugs - exceptions
else:
sources = all_slugs
for slug, config in all_sources.items():
# Backwards compatibility with specifying the name, rather than the slug,
# as the first arg in the definition
config.setdefault('name', slug)
slug = slugify(slug)
if slug not in sources:
log.debug('Skipping %s.' % slug)
continue
if (not options['reload']) and BoundarySet.objects.filter(slug=slug).exists():
log.info('Already loaded %s, skipping.' % slug)
continue
self.load_set(slug, config, options)
def test_autodiscover(self):
boundaries.registry = {}
boundaries._basepath = '.'
with LogCapture() as l:
boundaries.autodiscover('.')
self.assertEqual(boundaries.registry, {'Districts': {'file': './boundaries/tests/fixtures/', 'last_updated': date(2000, 1, 1)}})
l.check(('boundaries', 'WARNING', 'Multiple definitions of Districts found.'))
def test_autodiscover(self):
# Uses bar_definition.py and foo_definition.py fixtures.
boundaries.registry = {}
boundaries._basepath = '.'
with LogCapture() as l:
boundaries.autodiscover('./boundaries/tests/fixtures')
self.assertEqual(len(boundaries.registry), 1)
self.assertEqual(boundaries.registry['Districts']['file'], './boundaries/tests/fixtures/foo.shp')
self.assertEqual(boundaries.registry['Districts']['last_updated'], date(2000, 1, 1))
l.check(('boundaries', 'WARNING', 'Multiple definitions of Districts found.'))
def handle(self, *args, **options):
if settings.DEBUG:
print(
_('DEBUG is True. This can cause memory usage to balloon. Continue? [y/n]'
))
if input().lower() != 'y':
return
boundaries.autodiscover(options['data_dir'])
if options['only']:
whitelist = set(options['only'].split(','))
else:
whitelist = set()
if options['except']:
blacklist = set(options['except'].split(','))
else:
blacklist = set()
with alive_bar(len(boundaries.registry.items())) as bar:
for slug, definition in boundaries.registry.items():
bar()
name = slug
slug = slugify(slug)
if self.loadable(slug, definition['last_updated'], whitelist,
blacklist, options['reload']):
#log.info(_('Processing %(slug)s.') % {'slug': slug})
# Backwards-compatibility with having the name, instead of the slug,
# as the first argument to `boundaries.register`.
definition.setdefault('name', name)
definition = Definition(definition)
data_sources, tmpdirs = create_data_sources(
definition['file'],
encoding=definition['encoding'],
convert_3d_to_2d=options['clean'])
try:
if not data_sources:
log.warning(_('No shapefiles found.'))
else:
self.load_boundary_set(slug, definition,
data_sources, options)
finally:
for tmpdir in tmpdirs:
rmtree(tmpdir)
else:
log.debug(_('Skipping %(slug)s.') % {'slug': slug})
def handle(self, *args, **options):
if settings.DEBUG:
print(_("DEBUG is True. This can cause memory usage to balloon. Continue? [y/n]"))
if six.moves.input().lower() != "y":
return
boundaries.autodiscover(options["data_dir"])
if options["only"]:
whitelist = set(options["only"].split(","))
else:
whitelist = set()
if options["except"]:
blacklist = set(options["except"].split(","))
else:
blacklist = set()
for slug, definition in boundaries.registry.items():
name = slug
slug = slugify(slug)
if self.loadable(slug, definition["last_updated"], whitelist, blacklist, options["reload"]):
log.info(_("Processing %(slug)s.") % {"slug": slug})
# Backwards-compatibility with having the name, instead of the slug,
# as the first argument to `boundaries.register`.
definition.setdefault("name", name)
definition = Definition(definition)
data_sources, tmpdirs = create_data_sources(
definition["file"], definition["encoding"], options["clean"]
)
try:
if not data_sources:
log.warning(_("No shapefiles found."))
else:
self.load_boundary_set(slug, definition, data_sources, options)
finally:
for tmpdir in tmpdirs:
rmtree(tmpdir)
else:
log.debug(_("Skipping %(slug)s.") % {"slug": slug})
def handle(self, *args, **options):
if settings.DEBUG:
print('DEBUG is True - this can cause memory usage to balloon. continue? [y/n]')
if six.moves.input().lower() != 'y':
return
# Load configuration
boundaries.autodiscover(options['data_dir'])
all_sources = boundaries.registry
all_slugs = set(slugify(s) for s in all_sources)
if options['only']:
only = set(options['only'].split(','))
sources = only.intersection(all_slugs)
elif options['except']:
exceptions = set(options['except'].split(','))
sources = all_slugs - exceptions
else:
sources = all_slugs
for slug, config in all_sources.items():
# Backwards compatibility with specifying the name, rather than the slug,
# as the first arg in the definition
config.setdefault('name', slug)
slug = slugify(slug)
if slug not in sources:
log.debug('Skipping %s.' % slug)
continue
try:
existing_set = BoundarySet.objects.get(slug=slug)
if (not options['reload']) and existing_set.last_updated >= config['last_updated']:
log.info('Already loaded %s, skipping.' % slug)
continue
except BoundarySet.DoesNotExist:
pass
self.load_set(slug, config, options)
def handle(self, *args, **options):
if settings.DEBUG:
print(_('DEBUG is True. This can cause memory usage to balloon. Continue? [y/n]'))
if six.moves.input().lower() != 'y':
return
boundaries.autodiscover(options['data_dir'])
if options['only']:
whitelist = set(options['only'].split(','))
else:
whitelist = set()
if options['except']:
blacklist = set(options['except'].split(','))
else:
blacklist = set()
for slug, definition in boundaries.registry.items():
name = slug
slug = slugify(slug)
if self.loadable(slug, definition['last_updated'], whitelist, blacklist, options['reload']):
log.info(_('Processing %(slug)s.') % {'slug': slug})
# Backwards-compatibility with having the name, instead of the slug,
# as the first argument to `boundaries.register`.
definition.setdefault('name', name)
definition = Definition(definition)
data_sources, tmpdirs = create_data_sources(definition['file'], definition['encoding'], options['clean'])
try:
if not data_sources:
log.warning(_('No shapefiles found.'))
else:
self.load_boundary_set(slug, definition, data_sources, options)
finally:
for tmpdir in tmpdirs:
rmtree(tmpdir)
else:
log.debug(_('Skipping %(slug)s.') % {'slug': slug})
def handle(self, *args, **options):
boundaries.autodiscover(options['data_dir'])
for slug in sorted(boundaries.registry):
name = slug
slug = slugify(slug)
definition = boundaries.registry[name]
# Backwards-compatibility with having the name, instead of the slug,
# as the first argument to `boundaries.register`.
definition.setdefault('name', name)
definition = Definition(definition)
data_sources, tmpdirs = create_data_sources(definition['file'], encoding=definition['encoding'])
try:
if not data_sources:
log.warning(_('No shapefiles found.'))
else:
features = OrderedDict()
for data_source in data_sources:
features[slug] = []
layer = data_source[0]
for feature in layer:
feature = Feature(feature, definition)
if feature.is_valid():
features[slug].append((feature.id, feature.name))
for slug, features in features.items():
print('\n%s: %d' % (slug, len(features)))
for properties in sorted(features):
print('%s: %s' % properties)
finally:
for tmpdir in tmpdirs:
rmtree(tmpdir)
def handle(self, *args, **options):
boundaries.autodiscover(options['data_dir'])
for slug, definition in boundaries.registry.items():
name = slug
slug = slugify(slug)
# Backwards-compatibility with having the name, instead of the slug,
# as the first argument to `boundaries.register`.
definition.setdefault('name', name)
definition = Definition(definition)
data_sources, tmpdirs = create_data_sources(definition['file'], encoding=definition['encoding'])
try:
if not data_sources:
log.warning(_('No shapefiles found.'))
else:
features = OrderedDict()
for data_source in data_sources:
features[slug] = []
layer = data_source[0]
for feature in layer:
feature = Feature(feature, definition)
if feature.is_valid():
features[slug].append((feature.id, feature.name))
for slug, features in features.items():
print('\n%s: %d' % (slug, len(features)))
for properties in sorted(features):
print('%s: %s' % properties)
finally:
for tmpdir in tmpdirs:
rmtree(tmpdir)
def registry(base='.'): import boundaries boundaries.autodiscover(base) return boundaries.registry