class ElasticsearchBackend(BaseMetricsBackend):
def __init__(self,
hosts=None,
index="metrics",
doc_type="metric",
index_pattern="{index}-{date:%Y.%m.%d}",
*args,
**kwargs):
# Assign these in the backend as they are needed when writing metrics
# to elasticsearch
self.index = index
self.doc_type = doc_type
self.index_pattern = index_pattern
# setup the client
self.client = Elasticsearch(hosts=hosts, *args, **kwargs)
# ensure the index is created
try:
self._setup_index()
except TransportError as exc:
logger.error('index setup error %r', exc)
try:
self._setup_mapping()
except TransportError as exc:
logger.error('mapping setup error %r', exc)
def get_index(self):
return self.index_pattern.format(index=self.index, date=datetime.now())
def _setup_index(self):
return self.client.indices.create(self.get_index(), ignore=400)
def _setup_mapping(self):
return self.client.indices.put_template(
name="timeexecution-{}".format(self.index),
body={
"template": "{}*".format(self.index),
"mappings": {
self.doc_type: {
"dynamic_templates": [{
"strings": {
"mapping": {
"type": "keyword"
},
"match_mapping_type": "string"
}
}],
"_source": {
"enabled": True
},
"properties": {
"name": {
"type": "keyword"
},
"timestamp": {
"type": "date"
},
"hostname": {
"type": "keyword"
},
"value": {
"type": "float"
},
"origin": {
"type": "keyword"
},
}
},
},
"settings": {
"number_of_shards": "1",
"number_of_replicas": "1",
},
})
def write(self, name, **data):
"""
Write the metric to elasticsearch
Args:
name (str): The name of the metric to write
data (dict): Additional data to store with the metric
"""
data["name"] = name
if not ("timestamp" in data):
data["timestamp"] = datetime.utcnow()
try:
self.client.index(index=self.get_index(),
doc_type=self.doc_type,
id=None,
body=data)
except TransportError as exc:
logger.warning('writing metric %r failure %r', data, exc)
def bulk_write(self, metrics):
"""
Write multiple metrics to elasticsearch in one request
Args:
metrics (list): data with mappings to send to elasticsearch
"""
actions = []
index = self.get_index()
for metric in metrics:
actions.append(
{'index': {
'_index': index,
'_type': self.doc_type
}})
actions.append(metric)
try:
self.client.bulk(actions)
except TransportError as exc:
logger.warning('bulk_write metrics %r failure %r', metrics, exc)
from datetime import datetime
from elasticsearch6 import Elasticsearch
es = Elasticsearch()
# create an index in elasticsearch, ignore status code 400 (index already exists)
es.indices.create(index='my-index', ignore=400)
es.index(index="my-index",
id=42,
body={
"any": "data",
"timestamp": datetime.now()
})
data = json.loads(line)
casts = [
f'{key.replace("-", "").strip()}-{val.replace("-", "").strip()}'
for key, val in data['cast'].items()
]
try:
release_date, place = re.findall(r'(.+)\s\((.+)\)',
data['release_date'])[0]
data['release_date'] = str(
datetime.strptime(release_date,
'%d %B %Y').replace(day=1).date())
except ValueError:
try:
data['release_date'] = str(
datetime.strptime(release_date, '%B %Y').date())
except ValueError:
data['release_date'] = str(
datetime.strptime(release_date, '%Y').date())
except KeyError:
place = 'Unknown'
try:
runtime = re.findall(r'(\d+)\smin', data['runtime'])[0]
data['runtime'] = int(runtime)
except KeyError:
data.pop('runtime', None)
data['published_place'] = place
data['cast'] = casts
elasticsearch.index(index='imdb',
doc_type='imdb',
body=json.dumps(data).encode('utf-8'))