def scrapers_openspendingdoc(document_id):
config = load_config()
es = setup_elasticsearch(config)
kwargs = {
'config': config,
'document_id': document_id
}
OpenSpendingDocumentScraperRunner().run(**kwargs)
def scrapers_openspending(date_from, date_to):
config = load_config()
es = setup_elasticsearch(config)
try:
df = date_from.isoformat()
except AttributeError as e:
df = str(date_from)
try:
dt = date_to.isoformat()
except AttributeError as e:
dt = str(date_to)
kwargs = {
'config': config,
'date_from': df,
'date_to': dt
}
OpenSpendingScraperRunner().run(**kwargs)
def scrapers_poliflw(date_from, date_to, scroll):
config = load_config()
es = setup_elasticsearch(config)
try:
df = date_from.isoformat()
except AttributeError as e:
df = str(date_from)
try:
dt = date_to.isoformat()
except AttributeError as e:
dt = str(date_to)
kwargs = {
'config': config,
'date_from': df,
'date_to': dt,
'scroll': scroll
}
PoliflwScraperRunner().run(**kwargs)
def scrapers_obv_counts(date_from, date_to, threshold, organizations):
config = load_config()
es = setup_elasticsearch(config)
try:
df = date_from.isoformat()
except AttributeError as e:
df = str(date_from)
try:
dt = date_to.isoformat()
except AttributeError as e:
dt = str(date_to)
kwargs = {
'config': config,
'date_from': df,
'date_to': dt,
'threshold': threshold,
'organizations': organizations
}
OpenbesluitvormingCountsScraperRunner().run(**kwargs)
def es_put_template(template_dir):
"""
Put a template into Elasticsearch. A template contains settings and mappings
that should be applied to multiple indices. Check ``mappings/template.json``
for an example.
:param template_file: Path to JSON file containing the template. Defaults to ``mappings/template.json``.
"""
config = load_config()
es = setup_elasticsearch(config)
click.echo('Putting ES template from dir: %s' % template_dir)
for template_path in glob.glob(os.path.join(template_dir, 'es-*.json')):
click.echo(template_path)
template = {}
with open(template_path, 'rb') as template_file:
template = json.load(template_file)
template_name = os.path.basename(template_file.name.replace('es-','').replace('.json', ''))
es.indices.put_template(template_name, template)
index_name = 'jodal_%s' % (template_name)
if not es.indices.exists(index=index_name):
click.echo("Should make index %s" % (index_name,))
es.indices.create(index=index_name)
def scrapers_locations():
config = load_config()
es = setup_elasticsearch(config)
LocationsScraperRunner().run()