def geocode_collection(source_index, municipality_code):
print('\nGeocoding {} for municipality {}'.format(source_index,
municipality_code))
app = create_app({
'ELASTICSEARCH_HOST':
ELASTICSEARCH_HOST,
'CELERY_BROKER_URL':
'redis://' + os.getenv('REDIS_HOST', 'redis') + ':' +
os.getenv('REDIS_PORT', 6379) + '/' + os.getenv('REDIS_DB', 1),
})
# FIXME: monkeypatching settings may interact with flask app config
settings.ELASTICSEARCH_HOST = ELASTICSEARCH_HOST
waaroverheid_index = 'wo_{}'.format(municipality_code.lower())
source_count = es_source.count(index=source_index)['count']
try:
sink_count = es_sink.count(index=waaroverheid_index)['count']
except NotFoundError:
sink_count = 0
if source_count > sink_count:
latest_date, buckets = get_incomplete_buckets(source_index,
waaroverheid_index)
loaded_docs = []
for bucket in buckets:
bucket_docs = load_bucket(source_index, municipality_code,
latest_date, bucket)
loaded_docs += bucket_docs
# FIXME: percolate_documents never matches subscriptions with actual data
# TODO: try passing the right settings to celery for tasks.email_subscribers to work
# with app.app_context():
# percolate_documents(loaded_docs, latest_date, args.dry_run)
sink_count = es_sink.count(index=waaroverheid_index)['count']
else:
print('Skipping: source {} docs, sink {} docs'.format(
source_count, sink_count))
# update doc counts in MongoDB
for db_collection in ['municipality_highover', 'municipality_closeup']:
llv_db[db_collection].update_one(
{'properties.GM_CODE': municipality_code},
{'$set': {
'properties.doc_count': sink_count
}})
def main(args):
es = get_elasticsearch_connection()
app = create_app({'ELASTICSEARCH_HOST': ELASTICSEARCH_HOST})
for subscription in get_subscriptions(es):
doc_count, doc_sample = find_matching_docs(
subscription, args.loaded_since, es)
if doc_count > 0:
print("subscription {}: found {} docs, sending email to {}"
.format(subscription['_id'], doc_count, subscription['email']))
if not args.dry_run:
with app.app_context():
email_subscription(subscription, doc_count, args.loaded_since)
else:
print("subscription {}: no docs found, skipping"
.format(subscription['_id']))
import os.path
from werkzeug.wsgi import DispatcherMiddleware
from ocd_frontend import rest
application = DispatcherMiddleware(rest.create_app(),
{'/v0': rest.create_app()})
# For testing purposes, add a route that serves static files from a directory.
# DO NOT USE IN PRODUCTION. Serve static files through your webserver instead.
if application.app.config.get('DEBUG', False):
from flask import send_from_directory
@application.app.route('/data/<path:filename>')
def download_dump(filename):
collection_name = '_'.join(filename.split('_')[:2])
base_dir = os.path.join(application.app.config.get('DUMPS_DIR'),
collection_name)
return send_from_directory(base_dir, filename, as_attachment=True)
@application.app.route('/media/<path:filename>')
def serve_media(filename):
base_dir = os.path.join(application.app.config.get('THUMBNAILS_DIR'),
os.path.dirname(filename))
return send_from_directory(base_dir, os.path.basename(filename))
import os.path
from werkzeug.wsgi import DispatcherMiddleware
from ocd_frontend import rest
application = DispatcherMiddleware(rest.create_app(), {"/v0": rest.create_app()})
# For testing purposes, add a route that serves static files from a directory.
# DO NOT USE IN PRODUCTION. Serve static files through your webserver instead.
if application.app.config.get("DEBUG", False):
from flask import send_from_directory
@application.app.route("/data/<path:filename>")
def download_dump(filename):
collection_name = "_".join(filename.split("_")[:2])
base_dir = os.path.join(application.app.config.get("DUMPS_DIR"), collection_name)
return send_from_directory(base_dir, filename, as_attachment=True)
@application.app.route("/media/<path:filename>")
def serve_media(filename):
base_dir = os.path.join(application.app.config.get("THUMBNAILS_DIR"), os.path.dirname(filename))
return send_from_directory(base_dir, os.path.basename(filename))
from werkzeug.wsgi import DispatcherMiddleware
from ocd_frontend import rest
application = DispatcherMiddleware(rest.create_app(), {
'/v0': rest.create_app()
})