def es_cleanup():
"""
Delete documents beyond retention time
"""
config = load_config()
es = setup_elasticsearch(config)
for application in config['binoas']['applications']:
seconds = parse_frequency(
config['binoas']['applications'][application].get(
'retention', '1h'))
click.echo('Cleaning up %s (%s)' % (
application,
seconds,
))
es_query = {
"query": {
"range": {
"modified": {
"lt": "now-%ss/s" % (seconds, ),
}
}
}
}
index_name = 'binoas_%s' % (application, )
try:
res = es.delete_by_query(index=index_name,
doc_type='item',
body=es_query)
except NotFoundError:
res = None
print(res)
def database_rollback():
"""
Do a database rollback.
"""
config = load_config()
session = setup_db(config)
session.rollback()
def __init__(self, role, group=None):
"""
Initializes the process. The role is passed to give more information.
"""
multiprocessing.Process.__init__(self)
self.stop_event = multiprocessing.Event()
self.role = role
self.group = group
self.config = load_config()
def transform_json(json_file):
"""
Transform an incoming json document according to the contents.
"""
config = load_config()
template = json.load(json_file)
json_file.close()
trf = JSONPathPostTransformer(config)
res = trf.transform(template)
print(json.dumps(res))
def digest_make(frequency):
"""
Make a digest.
"""
config = load_config()
es = setup_elasticsearch(config)
digest = Digest(config)
for application in config['binoas']['applications']:
click.echo('Making digest for %s and frequency %s' %
(application, frequency))
digest.make(application, frequency)
sleep(5)
def test_load_config(self):
result = load_config('tests/data/test.yaml')
expected = {
'binoas': {
'zookeeper': 'kafka',
'_defaults': {
'routes': {
'app': {
'topics': {
'in': [
'topic'
]
},
},
'transformer': {
'topics': {
'in': [
'topic'
],
'out': [
'objects'
]
}
},
'loader': {
'topics': {
'in': [
'objects'
]
}
}
}
},
'applications': {
'poliflw': {
'name': 'PoliFLW'
},
'politwoops': {
'name': 'politwoops'
},
'openraadsinformatie': {
'name': "Open Raadsinformatie"
}
}
}
}
self.assertEqual(result, expected)
def es_put_template(template_file):
"""
Put a template into Elasticsearch. A template contains settings and mappings
that should be applied to multiple indices. Check ``mappings/template.json``
for an example.
:param template_file: Path to JSON file containing the template. Defaults to ``mappings/template.json``.
"""
config = load_config()
es = setup_elasticsearch(config)
click.echo('Putting ES template: %s' % template_file.name)
template = json.load(template_file)
template_file.close()
es.indices.put_template('binoas_template', template)
def create_app():
app = Flask(__name__)
app.config.from_object(Config)
app.config['binoas'] = load_config()['binoas']
app.errorhandler(BinoasError)(BinoasError.serialize_error)
def add_cors_headers(resp):
resp.headers['Access-Control-Allow-Origin'] = '*'
# See https://stackoverflow.com/questions/12630231/how-do-cors-and-access-control-allow-headers-work
resp.headers[
'Access-Control-Allow-Headers'] = 'origin, content-type, accept'
return resp
app.after_request(add_cors_headers)
setup_db(app.config)
setup_elasticsearch(app.config)
return app
def test_load_invalid_config(self):
with self.assertRaises(ConfigurationError):
load_config('tests/data/invalid.yaml')