def execute(self, ignore_cache=False, raise_on_error=True):
"""
Execute the multi search request and return a list of search results.
"""
if ignore_cache or not hasattr(self, '_response'):
es = connections.get_connection(self._using)
responses = es.msearch(index=self._index,
doc_type=self._get_doc_type(),
body=self.to_dict(),
**self._params)
out = []
for s, r in zip(self._searches, responses['responses']):
if r.get('error', False):
if raise_on_error:
raise TransportError('N/A', r['error']['type'],
r['error'])
r = None
else:
r = Response(s, r)
out.append(r)
self._response = out
return self._response
def execute(self, ignore_cache=False, raise_on_error=True):
"""
Execute the multi search request and return a list of search results.
"""
if ignore_cache or not hasattr(self, "_response"):
es = get_connection(self._using)
responses = es.msearch(index=self._index,
body=self.to_dict(),
**self._params)
out = []
for s, r in zip(self._searches, responses["responses"]):
if r.get("error", False):
if raise_on_error:
raise TransportError("N/A", r["error"]["type"],
r["error"])
r = None
else:
r = Response(s, r)
out.append(r)
self._response = out
return self._response
def test_perform_request_no_index(self, mock_log):
mock_log.level = logging.DEBUG
mock_log.debug.side_effect = TransportError(404,
'IndexMissingException')
conn = es.ESHttpConnection()
with pytest.raises(es.IndexNotFoundException):
conn.perform_request('POST', 'http://localhost:9200')
def execute(self, ignore_cache=False, raise_on_error=True):
if ignore_cache or not hasattr(self, '_response'):
es = connections.get_connection(self._using)
responses = es.msearch(index=self._index,
doc_type=self._doc_type,
body=self.to_dict(),
**self._params)
out = []
for s, r in zip(self._searches, responses['responses']):
if r.get('error', False):
print(r)
if raise_on_error:
raise TransportError('N/A', r['error']['type'],
r['error'])
r = None
else:
r = Response(r, callbacks=s._doc_type_map)
r.search = s
out.append(r)
self._response = out
return self._response
def _get_sniff_data(self, initial=False):
previous_sniff = self.last_sniff
try:
# reset last_sniff timestamp
self.last_sniff = self.loop.time()
for c in chain(
self.connection_pool.connections,
self.seed_connections,
):
try:
# use small timeout for the sniffing request,
# should be a fast api call
_, headers, node_info = yield from c.perform_request(
'GET',
'/_nodes/_all/http',
timeout=self.sniff_timeout if not initial else None,
)
node_info = self.deserializer.loads(
node_info,
headers.get('content-type'),
)
break
except (ConnectionError, SerializationError):
pass
else:
raise TransportError('N/A', 'Unable to sniff hosts.')
except: # noqa
# keep the previous value on error
self.last_sniff = previous_sniff
raise
return list(node_info['nodes'].values())
def test_transform_error_parse_with_error_string(self):
e = TransportError(500, "InternalServerError",
{"error": "something error message"})
self.assertEqual(
str(e),
"TransportError(500, 'InternalServerError', 'something error message')",
)
def test_selftest_failure_es(self, mocked_get_es):
"""
Elasticsearch is not reachable on selftest.
"""
mocked_get_es.return_value.info.side_effect = TransportError()
response = self.client.get(reverse('selftest'))
self.assertEquals(response.status_code, 500)
self.assertIn('es_error', response.data)
def test_bulk_write_error(self, mocked_logger):
transport_error = TransportError('mocked error')
es_index_error_ctx = mock.patch(
'time_execution.backends.elasticsearch.Elasticsearch.bulk',
side_effect=transport_error)
metrics = [1, 2, 3]
with es_index_error_ctx:
self.backend.bulk_write(metrics)
mocked_logger.warning.assert_called_once_with(
'bulk_write metrics %r failure %r', metrics, transport_error)
def test_transform_error_parse_with_error_reason(self):
e = TransportError(
500,
"InternalServerError",
{"error": {"root_cause": [{"type": "error", "reason": "error reason"}]}},
)
self.assertEqual(
str(e), "TransportError(500, 'InternalServerError', 'error reason')"
)
def test_search_es_exception(mocker, client, search_view):
"""If a 4xx status is returned from Elasticsearch it should be returned from the API"""
status_code = 418
search_mock = mocker.patch(
"search.views.execute_search",
autospec=True,
side_effect=TransportError(status_code),
)
query = {"query": {"match": {"title": "Search"}}}
resp = client.post(search_view.url, query)
assert resp.status_code == status_code
search_mock.assert_called_once_with(user=AnonymousUser(), query=query)
def test_related_posts_es_exception(mocker, client, related_posts_view):
"""If a 4xx status is returned from Elasticsearch it should be returned from the API"""
status_code = 418
related_documents_mock = mocker.patch(
"search.views.find_related_documents",
autospec=True,
side_effect=TransportError(status_code),
)
resp = client.post(related_posts_view.url)
assert resp.status_code == status_code
related_documents_mock.assert_called_once_with(
user=AnonymousUser(), post_id=related_posts_view.post_id)
def sniff_hosts(self, initial=False):
node_info = yield from self._get_sniff_data(initial)
hosts = list(filter(None, (self._get_host_info(n) for n in node_info)))
# we weren't able to get any nodes, maybe using an incompatible
# transport_schema or host_info_callback blocked all - raise error.
if not hosts:
raise TransportError(
'N/A',
'Unable to sniff hosts - no viable hosts found.',
)
self.set_connections(hosts)
def test_handle(self, mock_do, mock_log):
"""Test the main handle method calls do_index_command."""
obj = BaseSearchCommand()
obj.handle(indexes=["foo", "bar"])
# this should have called the do_index_command twice
mock_do.assert_has_calls([mock.call("foo"), mock.call("bar")])
mock_do.reset_mock()
mock_do.side_effect = TransportError(
123, "oops", {"error": {"reason": "no idea"}}
)
obj.handle(indexes=["baz"])
mock_do.assert_called_once_with("baz")
mock_log.warning.assert_called_once()
def _build_response(data):
out = []
for s, r in zip(self._searches, data['responses']):
if r.get('error', False):
if raise_on_error:
raise TransportError('N/A', r['error']['type'],
r['error'])
r = None
else:
r = Response(s, r)
out.append(r)
self._response = out
return self._response
def run(self):
"""
Run main metadata extraction suite.
"""
# Log beginning of processing
start = datetime.datetime.now()
self.logger.info("Metadata extraction started at: %s",
start.isoformat())
# Create index if necessary
if self.conf("send-to-index"):
es_factory = ElasticsearchClientFactory()
self.es = es_factory.get_client(self.configuration)
try:
index.create_index(self.configuration, self.es)
except TransportError as te:
if te[0] == 400:
pass
else:
raise TransportError(te)
if len(self.file_list) > 0:
# Process files
pool = []
for f in self.file_list:
# HDF libraries don't seem to like unicode strings,
# which the filenames will be if the configuration paths
# loaded from JSON end up in unicode
path = f
if "raw" not in path:
p = multiprocessing.Process(target=self.process_file,
args=(path,))
pool.append(p)
p.start()
while len(pool) >= self.conf("num-cores"):
for p in pool:
if p.exitcode is not None:
pool.remove(p)
for p in pool:
p.join()
# Log end of processing
end = datetime.datetime.now()
self.logger.info("Metadata extraction completed at: %s",
end.isoformat())
self.logger.info("Start: %s, End: %s, Total: %s",
start.isoformat(), end.isoformat(), end - start)
def test_handle(self, mock_do, mock_log):
"""Test the main handle method calls do_index_command."""
obj = BaseSearchCommand()
obj.handle(indexes=['foo', 'bar'])
# this should have called the do_index_command twice
mock_do.assert_has_calls([mock.call('foo'), mock.call('bar')])
mock_do.reset_mock()
mock_do.side_effect = TransportError(123, "oops",
{'error': {
'reason': 'no idea'
}})
obj.handle(indexes=['baz'])
mock_do.assert_called_once_with('baz')
mock_log.warning.assert_called_once()
def test_error_warning(self, mocked_logger):
transport_error = TransportError('mocked error')
es_index_error_ctx = mock.patch(
'time_execution.backends.elasticsearch.Elasticsearch.index',
side_effect=transport_error)
frozen_time_ctx = freeze_time('2016-07-13')
with es_index_error_ctx, frozen_time_ctx:
self.backend.write(name='test:metric', value=None)
mocked_logger.warning.assert_called_once_with(
'writing metric %r failure %r', {
'timestamp': datetime(2016, 7, 13),
'value': None,
'name': 'test:metric'
}, transport_error)
def test_rebuild_search_index(self, mock_update, mock_create, mock_delete):
"""Test the rebuild_search_index command."""
cmd = rebuild_search_index.Command()
result = cmd.do_index_command(
'foo', interactive=False) # True would hang the tests
mock_delete.assert_called_once_with('foo')
mock_create.assert_called_once_with('foo')
mock_update.assert_called_once_with('foo')
self.assertEqual(result['delete'], mock_delete.return_value)
self.assertEqual(result['create'], mock_create.return_value)
self.assertEqual(result['update'], mock_update.return_value)
# check that the delete is handled if the index does not exist
mock_delete.side_effect = TransportError("Index not found")
result = cmd.do_index_command(
'foo', interactive=False) # True would hang the tests
self.assertEqual(result['delete'], {})
async def execute(self, ignore_cache = False, raise_on_error = True):
if ignore_cache or not hasattr(self, '_response'):
responses = await self._using.msearch(index = self._index, doc_type = self._doc_type, body = self.to_dict(), **self._params)
out = []
for search, response in zip(self._searches, responses['responses']):
if response.get('error', False):
if raise_on_error:
raise TransportError('N/A', response['error']['type'], response['error'])
response = None
else:
response = Response(search, response)
out.append(response)
self._response = out
return self._response
def test_error_warning(self, mocked_logger):
transport_error = TransportError("mocked error")
es_index_error_ctx = mock.patch(
"time_execution.backends.elasticsearch.Elasticsearch.index",
side_effect=transport_error)
frozen_time_ctx = freeze_time("2016-07-13")
with es_index_error_ctx, frozen_time_ctx:
self.backend.write(name="test:metric", value=None)
mocked_logger.warning.assert_called_once_with(
"writing metric %r failure %r",
{
"timestamp": datetime(2016, 7, 13),
"value": None,
"name": "test:metric"
},
transport_error,
)
def test_sniff_on_fail_failing_does_not_prevent_retires(self, sniff_hosts):
sniff_hosts.side_effect = [TransportError("sniff failed")]
t = Transport(
[{
"exception": ConnectionError("abandon ship")
}, {
"data": CLUSTER_NODES
}],
connection_class=DummyConnection,
sniff_on_connection_fail=True,
max_retries=3,
randomize_hosts=False,
)
conn_err, conn_data = t.connection_pool.connections
response = t.perform_request("GET", "/")
self.assertEqual(json.loads(CLUSTER_NODES), response)
self.assertEqual(1, sniff_hosts.call_count)
self.assertEqual(1, len(conn_err.calls))
self.assertEqual(1, len(conn_data.calls))
def sniff_hosts(self, initial=False):
"""
Obtain a list of nodes from the cluster and create a new connection
pool using the information retrieved.
To extract the node connection parameters use the ``nodes_to_host_callback``.
:arg initial: flag indicating if this is during startup
(``sniff_on_start``), ignore the ``sniff_timeout`` if ``True``
"""
node_info = self._get_sniff_data(initial)
hosts = []
address_key = self.connection_class.transport_schema + '_address'
for n in node_info:
host = {}
address = n.get(address_key, '')
if '/' in address:
host['host'], address = address.split('/', 1)
# malformed address
if ':' not in address:
continue
ip, port = address.rsplit(':', 1)
# use the ip if not overridden by publish_host
host.setdefault('host', ip)
host['port'] = int(port)
host = self.host_info_callback(n, host)
if host is not None:
hosts.append(host)
# we weren't able to get any nodes, maybe using an incompatible
# transport_schema or host_info_callback blocked all - raise error.
if not hosts:
raise TransportError(
"N/A", "Unable to sniff hosts - no viable hosts found.")
self.set_connections(hosts)
def _get_sniff_data(self, initial=False):
"""
Perform the request to get sniffins information. Returns a list of
dictionaries (one per node) containing all the information from the
cluster.
It also sets the last_sniff attribute in case of a successful attempt.
In rare cases it might be possible to override this method in your
custom Transport class to serve data from alternative source like
configuration management.
"""
previous_sniff = self.last_sniff
try:
# reset last_sniff timestamp
self.last_sniff = time.time()
# go through all current connections as well as the
# seed_connections for good measure
for c in chain(self.connection_pool.connections,
self.seed_connections):
try:
# use small timeout for the sniffing request, should be a fast api call
_, headers, node_info = c.perform_request(
'GET',
'/_nodes/_all/clear',
timeout=self.sniff_timeout if not initial else None)
node_info = self.deserializer.loads(
node_info, headers.get('content-type'))
break
except (ConnectionError, SerializationError):
pass
else:
raise TransportError("N/A", "Unable to sniff hosts.")
except:
# keep the previous value on error
self.last_sniff = previous_sniff
raise
return list(node_info['nodes'].values())
async def test_sniff_on_fail_failing_does_not_prevent_retires(
self, sniff_hosts):
sniff_hosts.side_effect = [TransportError("sniff failed")]
t = AsyncTransport(
[{
"exception": ConnectionError("abandon ship")
}, {
"data": CLUSTER_NODES
}],
connection_class=DummyConnection,
sniff_on_connection_fail=True,
max_retries=3,
randomize_hosts=False,
)
await t._async_init()
conn_err, conn_data = t.connection_pool.connections
response = await t.perform_request("GET", "/")
assert json.loads(CLUSTER_NODES) == response
assert 1 == sniff_hosts.call_count
assert 1 == len(conn_err.calls)
assert 1 == len(conn_data.calls)
def __init__(self, config, threshold=1000):
"""
:param dict config: Application configuration dictionary, including ES config.
:param int threshold: The number of documents to hold in the buffer before indexing.
"""
self.index = config["es-index"]
self.default_mapping = config["es-mapping"]
self.threshold = threshold
self.es = Elasticsearch([_get_host_string(config)])
# If the index doesn't exist, create it
# This will throw an error if the index already exists this is *fine*
try:
create_index(config, self.es)
except TransportError as te:
if te[0] == 400:
print(te)
else:
raise TransportError(te)
# Dict containing key:value pairs of mapping:[list of documents]
# That way, this class can handle indexing multiple types of documents
self.doc_pool = {}
def test_partial_indexing(self, mock_exists, mock_read_json_file,
mock_ContentProcessor, mock_Elasticsearch,
mock_bulk):
"""
`sheer index --processors posts`
Test partial indexing of the document type associated with a
content processor.
"""
# Mock file existing/opening/reading
# os.path.exists is only called directly for settings.json and
# mappings.json, which are not necessary for our tests.
mock_exists.return_value = False
mock_read_json_file.side_effect = [self.mock_processors, {}]
# Wire-up our mock content processor
mock_ContentProcessor.return_value = self.mock_processor
# Here we want to test:
# * Index exists -> should be left alone
# * Mappings exist for processor -> should be deleted and recreated
# * Documents don't exist for processor -> should be created
mock_es = mock_Elasticsearch.return_value
mock_es.indices.exists.return_value = True
# The tests for the get_mapping return value simply need to evaluate to
# True in indexer.py.
mock_es.indices.get_mapping.return_value = True
mock_create_exception = TransportError(409)
mock_es.create.side_effect = mock_create_exception
test_args = AttrDict(processors=['posts'], reindex=False)
index_location(test_args, self.config)
mock_bulk.assert_called_with(mock_es,
self.mock_processor.documents(),
index='content')
def test_children_raises_on_404(self, mock_search):
mock_search().filter().filter().sort().extra(
).execute.side_effect = TransportError(404)
with self.assertRaises(TransportError):
IndexedFile.children('test_user', system='test.system', path='/')
def test_query_variants(self, mock_get_variants, mock_get_gene_counts,
mock_error_logger, mock_analyst_group):
url = reverse(query_variants_handler, args=['abc'])
self.check_collaborator_login(
url, request_data={'projectFamilies': PROJECT_FAMILIES})
url = reverse(query_variants_handler, args=[SEARCH_HASH])
# add a locus list
LocusList.objects.get(guid=LOCUS_LIST_GUID).projects.add(
Project.objects.get(guid=PROJECT_GUID))
# Test invalid inputs
response = self.client.get(url)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.reason_phrase,
'Invalid search hash: {}'.format(SEARCH_HASH))
mock_error_logger.assert_not_called()
response = self.client.post(url,
content_type='application/json',
data=json.dumps({'search': SEARCH}))
self.assertEqual(response.status_code, 400)
self.assertEqual(response.reason_phrase,
'Invalid search: no projects/ families specified')
mock_error_logger.assert_not_called()
mock_get_variants.side_effect = InvalidIndexException('Invalid index')
response = self.client.post(url,
content_type='application/json',
data=json.dumps({
'projectFamilies': PROJECT_FAMILIES,
'search': SEARCH
}))
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json()['error'], 'Invalid index')
mock_error_logger.assert_called_with('Invalid index', extra=mock.ANY)
mock_get_variants.side_effect = InvalidSearchException(
'Invalid search')
mock_error_logger.reset_mock()
response = self.client.post(url,
content_type='application/json',
data=json.dumps({
'projectFamilies': PROJECT_FAMILIES,
'search': SEARCH
}))
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json()['error'], 'Invalid search')
mock_error_logger.assert_not_called()
mock_get_variants.side_effect = ConnectionTimeout(
'', '', ValueError('Timeout'))
response = self.client.post(url,
content_type='application/json',
data=json.dumps({
'projectFamilies': PROJECT_FAMILIES,
'search': SEARCH
}))
self.assertEqual(response.status_code, 504)
self.assertEqual(response.json()['error'],
'ConnectionTimeout caused by - ValueError(Timeout)')
mock_error_logger.assert_not_called()
mock_get_variants.side_effect = TransportError(
'N/A', 'search_phase_execution_exception', {'error': 'Invalid'})
response = self.client.post(url,
content_type='application/json',
data=json.dumps({
'projectFamilies': PROJECT_FAMILIES,
'search': SEARCH
}))
self.assertEqual(response.status_code, 400)
self.assertEqual(
response.json()['error'],
"TransportError: N/A - 'search_phase_execution_exception' - 'Invalid'"
)
self.assertEqual(response.json()['detail'], {'error': 'Invalid'})
mock_error_logger.assert_not_called()
error_info_json = {
'error': {
'root_cause': [{
'type':
'response_handler_failure_transport_exception'
}]
}
}
mock_get_variants.side_effect = TransportError(
'401', 'search_phase_execution_exception', error_info_json)
response = self.client.post(url,
content_type='application/json',
data=json.dumps({
'projectFamilies': PROJECT_FAMILIES,
'search': SEARCH
}))
self.assertEqual(response.status_code, 401)
self.assertEqual(
response.json()['error'],
"TransportError: 401 - 'search_phase_execution_exception' - response_handler_failure_transport_exception"
)
self.assertEqual(response.json()['detail'], error_info_json)
mock_error_logger.assert_not_called()
mock_get_variants.side_effect = _get_es_variants
# Test new search
response = self.client.post(url,
content_type='application/json',
data=json.dumps({
'projectFamilies': PROJECT_FAMILIES,
'search': SEARCH
}))
self.assertEqual(response.status_code, 200)
response_json = response.json()
self.assertSetEqual(
set(response_json.keys()), {
'searchedVariants', 'savedVariantsByGuid', 'genesById',
'search', 'variantTagsByGuid', 'variantNotesByGuid',
'variantFunctionalDataByGuid', 'locusListsByGuid'
})
self.assertListEqual(response_json['searchedVariants'], VARIANTS)
self.assertDictEqual(
response_json['search'], {
'search':
SEARCH,
'projectFamilies': [{
'projectGuid': PROJECT_GUID,
'familyGuids': mock.ANY
}],
'totalResults':
3,
})
self.assertSetEqual(
set(response_json['search']['projectFamilies'][0]['familyGuids']),
{'F000001_1', 'F000002_2'})
self.assertSetEqual(set(response_json['savedVariantsByGuid'].keys()), {
'SV0000001_2103343353_r0390_100', 'SV0000002_1248367227_r0390_100'
})
self.assertSetEqual(
set(response_json['genesById'].keys()),
{'ENSG00000227232', 'ENSG00000268903', 'ENSG00000233653'})
gene_fields = {'locusListGuids'}
gene_fields.update(GENE_VARIANT_FIELDS)
self.assertSetEqual(
set(response_json['genesById']['ENSG00000227232'].keys()),
gene_fields)
self.assertListEqual(
response_json['genesById']['ENSG00000227232']['locusListGuids'],
[LOCUS_LIST_GUID])
self.assertSetEqual(set(response_json['locusListsByGuid'].keys()),
{LOCUS_LIST_GUID})
intervals = response_json['locusListsByGuid'][LOCUS_LIST_GUID][
'intervals']
self.assertEqual(len(intervals), 2)
self.assertSetEqual(
set(intervals[0].keys()), {
'locusListGuid', 'locusListIntervalGuid', 'genomeVersion',
'chrom', 'start', 'end'
})
results_model = VariantSearchResults.objects.get(
search_hash=SEARCH_HASH)
mock_get_variants.assert_called_with(results_model,
sort='xpos',
page=1,
num_results=100,
skip_genotype_filter=False)
mock_error_logger.assert_not_called()
# Test pagination
response = self.client.get('{}?page=3'.format(url))
self.assertEqual(response.status_code, 200)
mock_get_variants.assert_called_with(results_model,
sort='xpos',
page=3,
num_results=100,
skip_genotype_filter=False)
mock_error_logger.assert_not_called()
# Test sort
response = self.client.get('{}?sort=pathogenicity'.format(url))
self.assertEqual(response.status_code, 200)
mock_get_variants.assert_called_with(results_model,
sort='pathogenicity',
page=1,
num_results=100,
skip_genotype_filter=False)
mock_error_logger.assert_not_called()
# Test export
export_url = reverse(export_variants_handler, args=[SEARCH_HASH])
response = self.client.get(export_url)
self.assertEqual(response.status_code, 200)
expected_content = [
[
'chrom', 'pos', 'ref', 'alt', 'gene', 'worst_consequence',
'1kg_freq', 'exac_freq', 'gnomad_genomes_freq',
'gnomad_exomes_freq', 'topmed_freq', 'cadd', 'revel', 'eigen',
'polyphen', 'sift', 'muttaster', 'fathmm', 'rsid', 'hgvsc',
'hgvsp', 'clinvar_clinical_significance', 'clinvar_gold_stars',
'filter', 'family_id_1', 'tags_1', 'notes_1', 'family_id_2',
'tags_2', 'notes_2', 'sample_1', 'num_alt_alleles_1', 'gq_1',
'ab_1', 'sample_2', 'num_alt_alleles_2', 'gq_2', 'ab_2'
],
[
'21', '3343400', 'GAGA', 'G', 'WASH7P', 'missense_variant', '',
'', '', '', '', '', '', '', '', '', '', '', '',
'ENST00000623083.3:c.1075G>A', 'ENSP00000485442.1:p.Gly359Ser',
'', '', '', '1',
'Tier 1 - Novel gene and phenotype (None)|Review (None)', '',
'2', '', '', 'NA19675', '1', '46.0', '0.702127659574',
'NA19679', '0', '99.0', '0.0'
],
[
'3', '835', 'AAAG', 'A', '', '', '', '', '', '', '', '', '',
'', '', '', '', '', '', '', '', '', '', '', '1', '', '', '',
'', '', 'NA19679', '0', '99.0', '0.0', '', '', '', ''
],
[
'12', '48367227', 'TC', 'T', '', '', '', '', '', '', '', '',
'', '', '', '', '', '', '', '', '', '', '', '', '2',
'Known gene for phenotype (None)|Excluded (None)',
'test n\xf8te (None)', '', '', '', '', '', '', '', '', '', '',
''
]
]
self.assertEqual(
response.content,
('\n'.join(['\t'.join(line)
for line in expected_content]) + '\n').encode('utf-8'))
mock_get_variants.assert_called_with(results_model,
page=1,
load_all=True)
mock_error_logger.assert_not_called()
# Test gene breakdown
gene_counts = {
'ENSG00000227232': {
'total': 2,
'families': {
'F000001_1': 2,
'F000002_2': 1
}
},
'ENSG00000268903': {
'total': 1,
'families': {
'F000002_2': 1
}
}
}
mock_get_gene_counts.return_value = gene_counts
gene_breakdown_url = reverse(get_variant_gene_breakdown,
args=[SEARCH_HASH])
response = self.client.get(gene_breakdown_url)
self.assertEqual(response.status_code, 200)
response_json = response.json()
self.assertSetEqual(set(response_json.keys()),
{'searchGeneBreakdown', 'genesById'})
self.assertDictEqual(response_json['searchGeneBreakdown'],
{SEARCH_HASH: gene_counts})
self.assertSetEqual(set(response_json['genesById'].keys()),
{'ENSG00000227232', 'ENSG00000268903'})
gene_fields = {
'constraints', 'omimPhenotypes', 'mimNumber', 'cnSensitivity'
}
gene_fields.update(GENE_FIELDS)
self.assertSetEqual(
set(response_json['genesById']['ENSG00000227232'].keys()),
gene_fields)
# Test compound hets
mock_get_variants.side_effect = _get_compound_het_es_variants
response = self.client.post(url,
content_type='application/json',
data=json.dumps({
'projectFamilies': PROJECT_FAMILIES,
'search': SEARCH
}))
self.assertEqual(response.status_code, 200)
response_json = response.json()
self.assertSetEqual(
set(response_json.keys()), {
'searchedVariants', 'savedVariantsByGuid', 'genesById',
'search', 'variantTagsByGuid', 'variantNotesByGuid',
'variantFunctionalDataByGuid', 'locusListsByGuid'
})
self.assertListEqual(response_json['searchedVariants'],
COMP_HET_VARAINTS)
self.assertSetEqual(set(response_json['savedVariantsByGuid'].keys()),
{'SV0000002_1248367227_r0390_100'})
self.assertSetEqual(set(response_json['genesById'].keys()),
{'ENSG00000233653'})
mock_error_logger.assert_not_called()
# Test cross-project discovery for analyst users
self.login_analyst_user()
mock_get_variants.side_effect = _get_es_variants
response = self.client.get('{}?sort=pathogenicity'.format(url))
self.assertEqual(response.status_code, 403)
mock_analyst_group.__bool__.return_value = True
mock_analyst_group.resolve_expression.return_value = 'analysts'
response = self.client.get('{}?sort=pathogenicity'.format(url))
self.assertEqual(response.status_code, 200)
response_json = response.json()
self.assertSetEqual(
set(response_json.keys()), {
'searchedVariants', 'savedVariantsByGuid', 'genesById',
'search', 'variantTagsByGuid', 'variantNotesByGuid',
'variantFunctionalDataByGuid', 'familiesByGuid',
'locusListsByGuid'
})
self.assertListEqual(response_json['searchedVariants'],
VARIANTS_WITH_DISCOVERY_TAGS)
self.assertSetEqual(set(response_json['familiesByGuid'].keys()),
{'F000011_11'})
mock_get_variants.assert_called_with(results_model,
sort='pathogenicity_hgmd',
page=1,
num_results=100,
skip_genotype_filter=False)
mock_error_logger.assert_not_called()
# Test no results
mock_get_variants.side_effect = _get_empty_es_variants
response = self.client.post(url,
content_type='application/json',
data=json.dumps({
'projectFamilies': PROJECT_FAMILIES,
'search': SEARCH
}))
self.assertEqual(response.status_code, 200)
response_json = response.json()
self.assertDictEqual(
response_json, {
'searchedVariants': [],
'search': {
'search': SEARCH,
'projectFamilies': PROJECT_FAMILIES,
'totalResults': 0,
}
})
mock_error_logger.assert_not_called()
def test_from_path_with_404(self, mock_refresh, mock_search):
mock_search().filter().execute.side_effect = TransportError(404)
with self.assertRaises(TransportError):
IndexedFile.from_path('test.system', '/')
def test_perform_request_exception(self):
conn = es.ESHttpConnection()
conn.pool = Mock()
conn.pool.urlopen.side_effect = TransportError('N/A', '')
with pytest.raises(JHTTPBadRequest):
conn.perform_request('POST', 'http://localhost:9200')