def get_indices(self, dates):
"""Return the list of indices to use for given dates. """
start_date = None
end_date = None
for date in dates:
if '>' in date.operator:
start_date = date.value
if '<' in date.operator:
end_date = date.value
return generate_list_of_indexes(start_date, end_date, self.context.get_index_template())
def test_get_with_indices(self, mocked_es):
mocked_connection = mock.Mock()
mocked_es.Elasticsearch.return_value = mocked_connection
# Test default indices.
self.api.get(
query={'query': {}}
)
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=[self.es_context.get_index_template()],
doc_type=self.es_context.get_doctype()
)
# Test all indices.
self.api.get(
query={'query': {}},
indices=['ALL']
)
mocked_connection.search.assert_called_with(
body='{"query": {}}'
)
# Test forcing indices.
self.api.get(
query={'query': {}},
indices=['socorro_201801', 'socorro_200047', 'not_an_index']
)
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=['socorro_201801', 'socorro_200047', 'not_an_index'],
doc_type=self.es_context.get_doctype()
)
# Test default indices with an index schema based on dates.
index_schema = 'socorro_%Y%W'
config = self.get_base_config(cls=Query, es_index=index_schema)
api = Query(config=config)
now = datetimeutil.utc_now()
last_week = now - datetime.timedelta(days=7)
indices = generate_list_of_indexes(last_week, now, api.context.get_index_template())
api.get(
query={'query': {}}
)
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=indices,
doc_type=api.context.get_doctype()
)
def main(self):
context = self.config.elasticsearch_class(self.config)
index_name_template = context.get_index_template()
# Figure out dates
today = date.today()
from_date = today - timedelta(weeks=self.config.weeks_to_create_past)
to_date = today + timedelta(weeks=self.config.weeks_to_create_future)
# Create indexes
index_names = generate_list_of_indexes(from_date, to_date,
index_name_template)
for index_name in index_names:
context.create_socorro_index(index_name, log_result=True)
def main(self):
index_creator = self.config.index_creator_class(self.config)
index_name_template = self.config.elasticsearch.elasticsearch_index
# Figure out dates
today = date.today()
from_date = today - timedelta(weeks=self.config.weeks_to_create_past)
to_date = today + timedelta(weeks=self.config.weeks_to_create_future)
# Create indexes
index_names = generate_list_of_indexes(from_date, to_date,
index_name_template)
for index_name in index_names:
index_creator.create_socorro_index(index_name)
return SUCCESS
def test_get_with_indices(self, mocked_es):
mocked_connection = mock.Mock()
mocked_es.Elasticsearch.return_value = mocked_connection
# Test indices with dates (the test configuration includes dates).
self.api.get(query={"query": {}})
now = utc_now()
last_week = now - datetime.timedelta(days=7)
indices = generate_list_of_indexes(
last_week, now, self.api.context.get_index_template())
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=indices,
doc_type=self.es_context.get_doctype(),
)
# Test all indices.
self.api.get(query={"query": {}}, indices=["ALL"])
mocked_connection.search.assert_called_with(body='{"query": {}}')
# Test forcing indices.
self.api.get(
query={"query": {}},
indices=["socorro_201801", "socorro_200047", "not_an_index"],
)
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=["socorro_201801", "socorro_200047", "not_an_index"],
doc_type=self.es_context.get_doctype(),
)
# Test default indices with an index schema based on dates.
index_schema = "testsocorro"
config = self.get_base_config(cls=Query, es_index=index_schema)
api = Query(config=config)
api.get(query={"query": {}})
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=["testsocorro"],
doc_type=api.context.get_doctype(),
)
def get(self, **kwargs):
"""Return the result of a custom query"""
params = external_common.parse_arguments(self.filters, kwargs)
if not params.query:
raise MissingArgumentError('query')
# Set indices.
indices = []
if not params.indices:
# By default, use the last two indices.
today = datetimeutil.utc_now()
last_week = today - datetime.timedelta(days=7)
index_template = self.config.elasticsearch.elasticsearch_index
indices = generate_list_of_indexes(last_week, today,
index_template)
elif len(params.indices) == 1 and params.indices[0] == 'ALL':
# If we want all indices, just do nothing.
pass
else:
indices = params.indices
search_args = {}
if indices:
search_args['index'] = indices
search_args['doc_type'] = (
self.config.elasticsearch.elasticsearch_doctype)
connection = self.get_connection()
try:
results = connection.search(body=json.dumps(params.query),
**search_args)
except elasticsearch.exceptions.NotFoundError as e:
missing_index = re.findall(BAD_INDEX_REGEX, e.error)[0]
raise ResourceNotFound("elasticsearch index '%s' does not exist" %
missing_index)
except elasticsearch.exceptions.TransportError as e:
raise DatabaseError(e)
return results
def cmd_create(weeks_past, weeks_future):
"""Create recent indices."""
conn = get_conn()
# Create recent indices
index_name_template = conn.get_index_template()
# Figure out dates
today = datetime.date.today()
from_date = today - datetime.timedelta(weeks=weeks_past)
to_date = today + datetime.timedelta(weeks=weeks_future)
# Create indiices
index_names = generate_list_of_indexes(from_date, to_date, index_name_template)
for index_name in index_names:
was_created = conn.create_index(index_name)
if was_created:
print("Index %s was created." % index_name)
else:
print("Index %s already existed." % index_name)
def cmd_create(weeks_past, weeks_future):
"""Create recent indices."""
conn = get_conn()
# Create recent indices
index_name_template = conn.get_index_template()
# Figure out dates
today = datetime.date.today()
from_date = today - datetime.timedelta(weeks=weeks_past)
to_date = today + datetime.timedelta(weeks=weeks_future)
# Create indiices
index_names = generate_list_of_indexes(from_date, to_date, index_name_template)
for index_name in index_names:
was_created = conn.create_index(index_name)
if was_created:
print('Index %s was created.' % index_name)
else:
print('Index %s already existed.' % index_name)
def test_get_with_indices(self, mocked_es):
mocked_connection = mock.Mock()
mocked_es.Elasticsearch.return_value = mocked_connection
# Test default indices.
self.api.get(query={'query': {}})
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=[self.es_context.get_index_template()],
doc_type=self.es_context.get_doctype())
# Test all indices.
self.api.get(query={'query': {}}, indices=['ALL'])
mocked_connection.search.assert_called_with(body='{"query": {}}')
# Test forcing indices.
self.api.get(
query={'query': {}},
indices=['socorro_201801', 'socorro_200047', 'not_an_index'])
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=['socorro_201801', 'socorro_200047', 'not_an_index'],
doc_type=self.es_context.get_doctype())
# Test default indices with an index schema based on dates.
index_schema = 'socorro_%Y%W'
config = self.get_base_config(cls=Query, es_index=index_schema)
api = Query(config=config)
now = datetimeutil.utc_now()
last_week = now - datetime.timedelta(days=7)
indices = generate_list_of_indexes(last_week, now,
api.context.get_index_template())
api.get(query={'query': {}})
mocked_connection.search.assert_called_with(
body='{"query": {}}',
index=indices,
doc_type=api.context.get_doctype())
def get(self, **kwargs):
"""Return the result of a custom query"""
params = external_common.parse_arguments(self.filters, kwargs)
if not params.query:
raise MissingArgumentError('query')
# Set indices.
indices = []
if not params.indices:
# By default, use the last two indices.
today = datetimeutil.utc_now()
last_week = today - datetime.timedelta(days=7)
index_template = self.context.get_index_template()
indices = generate_list_of_indexes(last_week, today, index_template)
elif len(params.indices) == 1 and params.indices[0] == 'ALL':
# If we want all indices, just do nothing.
pass
else:
indices = params.indices
search_args = {}
if indices:
search_args['index'] = indices
search_args['doc_type'] = self.context.get_doctype()
connection = self.get_connection()
try:
results = connection.search(body=json.dumps(params.query), **search_args)
except elasticsearch.exceptions.NotFoundError as e:
missing_index = re.findall(BAD_INDEX_REGEX, e.error)[0]
raise ResourceNotFound("elasticsearch index '%s' does not exist" % missing_index)
except elasticsearch.exceptions.TransportError as e:
raise DatabaseError(e)
return results
