def test_safe_redis_get_json(self, mock_redis, mock_logger):
# test with valid json
mock_redis.return_value.get.side_effect = lambda key: json.dumps(
{key: 'test'})
self.assertDictEqual(safe_redis_get_json('test_key'),
{'test_key': 'test'})
mock_logger.info.assert_called_with('Loaded test_key from redis')
mock_logger.warn.assert_not_called()
# test with no value in cache
mock_logger.reset_mock()
mock_redis.return_value.get.side_effect = lambda key: None
self.assertIsNone(safe_redis_get_json('test_key'))
mock_logger.info.assert_not_called()
mock_logger.warn.assert_not_called()
# test with invalid json in cache
mock_logger.reset_mock()
mock_redis.return_value.get.side_effect = lambda key: key
self.assertIsNone(safe_redis_get_json('test_key'))
mock_logger.info.assert_called_with('Loaded test_key from redis')
self.assertEqual(mock_logger.warn.call_args.args[0].split('\t')[0],
'Unable to fetch "test_key" from redis:')
# test with redis connection error
mock_logger.reset_mock()
mock_redis.side_effect = Exception('invalid redis')
self.assertIsNone(safe_redis_get_json('test_key'))
mock_logger.info.assert_not_called()
mock_logger.warn.assert_called_with(
'Unable to connect to redis host localhost: invalid redis')
def get_index_metadata(index_name,
client,
include_fields=False,
use_cache=True):
if use_cache:
cache_key = 'index_metadata__{}'.format(index_name)
cached_metadata = safe_redis_get_json(cache_key)
if cached_metadata:
return cached_metadata
try:
mappings = client.indices.get_mapping(index=index_name)
except Exception as e:
raise InvalidIndexException('{} - Error accessing index: {}'.format(
index_name, e.error if hasattr(e, 'error') else str(e)))
index_metadata = {}
for index_name, mapping in mappings.items():
variant_mapping = mapping['mappings']
index_metadata[index_name] = variant_mapping.get('_meta', {})
if include_fields:
index_metadata[index_name]['fields'] = {
field: field_props.get('type')
for field, field_props in
variant_mapping['properties'].items()
}
if use_cache and include_fields:
# Only cache metadata with fields
safe_redis_set_json(cache_key, index_metadata)
return index_metadata
def get_project_guids_user_can_view(user):
cache_key = 'projects__{}'.format(user)
project_guids = safe_redis_get_json(cache_key)
if project_guids is not None:
return project_guids
is_data_manager = user_is_data_manager(user)
if is_data_manager:
projects = Project.objects.all()
else:
projects = get_local_access_projects(user)
project_guids = [p.guid for p in projects.only('guid')]
if is_anvil_authenticated(user) and not is_data_manager:
workspaces = [
'/'.join([ws['workspace']['namespace'], ws['workspace']['name']])
for ws in list_anvil_workspaces(user)
]
project_guids += [
p.guid for p in Project.objects.filter(
workspace_name__isnull=False).exclude(
workspace_name='').exclude(
guid__in=project_guids).annotate(
workspace=Concat('workspace_namespace', Value(
'/'), 'workspace_name')).filter(
workspace__in=workspaces).only('guid')
]
safe_redis_set_json(cache_key,
project_guids,
expire=TERRA_WORKSPACE_CACHE_EXPIRE_SECONDS)
return project_guids
def list_anvil_workspaces(user):
"""Get all the workspaces accessible by the logged-in user.
:param
user (User model): who's credentials will be used to access AnVIL
:return
A list of workspaces that the user has access (OWNER, WRITER, or READER). Each of the workspace has
its name and namespace.
"""
path = 'api/workspaces?fields=public,workspace.name,workspace.namespace'
cache_key = 'terra_req__{}__{}'.format(user, path)
r = safe_redis_get_json(cache_key)
if r:
logger.info('Terra API cache hit for: GET {} {}'.format(path, user))
return r
r = _user_anvil_call('get', path, user)
# remove the public workspaces which can't be the projects in seqr
r = [{'workspace': ws['workspace']} for ws in r if not ws.get('public', True)]
safe_redis_set_json(cache_key, r, TERRA_WORKSPACE_CACHE_EXPIRE_SECONDS)
return r
def user_get_workspace_access_level(user, workspace_namespace, workspace_name, meta_fields=None):
fields = ',{}'.format(','.join(meta_fields)) if meta_fields else ''
path = "api/workspaces/{0}/{1}?fields=accessLevel,canShare{2}".format(workspace_namespace, workspace_name, fields)
cache_key = 'terra_req__{}__{}'.format(user, path)
r = safe_redis_get_json(cache_key)
if r:
logger.info('Terra API cache hit for: GET {} {}'.format(path, user))
return r
# Exceptions are handled to return an empty result for users who have no permission to access the workspace
r = _user_anvil_call('get', path, user, handle_errors=True)
if r:
safe_redis_set_json(cache_key, r, TERRA_PERMS_CACHE_EXPIRE_SECONDS)
return r
def user_get_workspace_access_level(user, workspace_namespace, workspace_name):
path = "api/workspaces/{0}/{1}?fields=accessLevel".format(
workspace_namespace, workspace_name)
cache_key = 'terra_req__{}__{}'.format(user, path)
r = safe_redis_get_json(cache_key)
if r:
logger.info('Terra API cache hit for: GET {} {}'.format(path, user))
return r
try:
r = _user_anvil_call('get', path, user)
except TerraNotFoundException as et:
logger.warning(str(et))
return {}
safe_redis_set_json(cache_key, r, TERRA_PERMS_CACHE_EXPIRE_SECONDS)
return r
def get_index_metadata(index_name, client):
cache_key = 'index_metadata__{}'.format(index_name)
cached_metadata = safe_redis_get_json(cache_key)
if cached_metadata:
return cached_metadata
try:
mappings = client.indices.get_mapping(index=index_name)
except Exception as e:
raise InvalidIndexException('Error accessing index "{}": {}'.format(
index_name, e.error if hasattr(e, 'error') else e.message))
index_metadata = {}
for index_name, mapping in mappings.items():
variant_mapping = mapping['mappings'].get(
VARIANT_DOC_TYPE) or mapping['mappings'].get(SV_DOC_TYPE, {})
index_metadata[index_name] = variant_mapping.get('_meta', {})
index_metadata[index_name]['fields'] = variant_mapping[
'properties'].keys()
safe_redis_set_json(cache_key, index_metadata)
return index_metadata
def get_es_variants(search_model,
es_search_cls=EsSearch,
sort=XPOS_SORT_KEY,
**kwargs):
cache_key = 'search_results__{}__{}'.format(search_model.guid, sort
or XPOS_SORT_KEY)
previous_search_results = safe_redis_get_json(cache_key) or {}
previously_loaded_results, search_kwargs = es_search_cls.process_previous_results(
previous_search_results, **kwargs)
if previously_loaded_results is not None:
return previously_loaded_results, previous_search_results.get(
'total_results')
search = search_model.variant_search.search
genes, intervals, invalid_items = parse_locus_list_items(
search.get('locus', {}))
if invalid_items:
raise Exception('Invalid genes/intervals: {}'.format(
', '.join(invalid_items)))
rs_ids, variant_ids, invalid_items = _parse_variant_items(
search.get('locus', {}))
if invalid_items:
raise Exception('Invalid variants: {}'.format(
', '.join(invalid_items)))
es_search = es_search_cls(
search_model.families.all(),
previous_search_results=previous_search_results,
skip_unaffected_families=search.get('inheritance'),
)
if search.get('customQuery'):
custom_q = search['customQuery']
if not isinstance(custom_q, list):
custom_q = [custom_q]
for q_dict in custom_q:
es_search.filter(Q(q_dict))
if sort:
es_search.sort(sort)
if genes or intervals or rs_ids or variant_ids:
es_search.filter_by_location(genes=genes,
intervals=intervals,
rs_ids=rs_ids,
variant_ids=variant_ids,
locus=search['locus'])
if (variant_ids or rs_ids) and not (
genes
or intervals) and not search['locus'].get('excludeLocations'):
search_kwargs['num_results'] = len(variant_ids) + len(rs_ids)
if search.get('freqs'):
es_search.filter_by_frequency(search['freqs'])
es_search.filter_by_annotation_and_genotype(
search.get('inheritance'),
quality_filter=search.get('qualityFilter'),
annotations=search.get('annotations'),
annotations_secondary=search.get('annotations_secondary'),
pathogenicity=search.get('pathogenicity'))
if hasattr(es_search, 'aggregate_by_gene'):
es_search.aggregate_by_gene()
variant_results = es_search.search(**search_kwargs)
safe_redis_set_json(cache_key, es_search.previous_search_results)
return variant_results, es_search.previous_search_results['total_results']
