def test_create_idx_type_model_permissions(self):
elastic_settings_before = ElasticSettings.attrs().get('IDX')
user_types_before = elastic_settings_before['CP_STATS_UD']['idx_type']
self.assertEqual({}, user_types_before,
'CP_STATS_UD idx_type is empty')
idx = "cp:hg19_userdata_bed"
new_upload_file = "tmp_newly_uploaded_file"
idx_type = new_upload_file
os.system(
"curl -XPUT " + ElasticSettings.url() + "/" + idx + "/_mapping/" +
idx_type + " -d '{\"" + idx_type +
"\":{ \"properties\" : {\"message\" : {\"type\" : \"string\", \"store\" : true } } }}'"
)
os.system("curl -XPUT " + ElasticSettings.url() + "/" + idx + "/" +
idx_type + "/_meta -d '{\"label\": \"" + new_upload_file +
"\", \"owner\": \"" + self.user.username +
"\", \"uploaded\": \"" + str(timezone.now()) + "\"}'")
elastic_settings_after = elastic_factory.create_idx_type_model_permissions(
self.user,
indexKey='CP_STATS_UD',
indexTypeKey='UD-' + new_upload_file.upper(), # @IgnorePep8
new_upload_file="tmp_newly_uploaded_file") # @IgnorePep8
# elastic_settings_after = elastic_factory.get_elastic_settings_with_user_uploads(elastic_settings_before)
user_types_after = elastic_settings_after['CP_STATS_UD']['idx_type']
self.assertTrue(len(user_types_after) > 0, "Has user idx_types ")
self.assertTrue('UD-TMP_NEWLY_UPLOADED_FILE' in user_types_after)
self.assertEqual(
user_types_after['UD-TMP_NEWLY_UPLOADED_FILE']['type'],
'tmp_newly_uploaded_file')
def test_create_idx_type_model_permissions(self):
elastic_settings_before = ElasticSettings.attrs().get('IDX')
user_types_before = elastic_settings_before['CP_STATS_UD']['idx_type']
self.assertEqual({}, user_types_before, 'CP_STATS_UD idx_type is empty')
idx = "cp:hg19_userdata_bed"
new_upload_file = "tmp_newly_uploaded_file"
idx_type = new_upload_file
os.system("curl -XPUT "+ElasticSettings.url()+"/"+idx+"/_mapping/"+idx_type+" -d '{\"" +
idx_type + "\":{ \"properties\" : {\"message\" : {\"type\" : \"string\", \"store\" : true } } }}'")
os.system("curl -XPUT "+ElasticSettings.url()+"/"+idx+"/"+idx_type+"/_meta -d '{\"label\": \"" +
new_upload_file + "\", \"owner\": \""+self.user.username+"\", \"uploaded\": \"" +
str(timezone.now())+"\"}'")
elastic_settings_after = elastic_factory.create_idx_type_model_permissions(self.user,
indexKey='CP_STATS_UD',
indexTypeKey='UD-'+new_upload_file.upper(), # @IgnorePep8
new_upload_file="tmp_newly_uploaded_file") # @IgnorePep8
# elastic_settings_after = elastic_factory.get_elastic_settings_with_user_uploads(elastic_settings_before)
user_types_after = elastic_settings_after['CP_STATS_UD']['idx_type']
self.assertTrue(len(user_types_after) > 0, "Has user idx_types ")
self.assertTrue('UD-TMP_NEWLY_UPLOADED_FILE' in user_types_after)
self.assertEqual(user_types_after['UD-TMP_NEWLY_UPLOADED_FILE']['type'], 'tmp_newly_uploaded_file')
def get_idx_and_idx_type_keys(cls, elastic_dict=None, auth_public=False):
'''
Returns the idx and idx type keys
Use the auth_public flag to restrict to either public or private idx and idx_types
'''
if elastic_dict is None:
elastic_dict = ElasticSettings.attrs().get('IDX')
connection = connections[settings.AUTH_DB]
if settings.INCLUDE_USER_UPLOADS is True:
if "django_content_type" in connection.introspection.table_names(
):
elastic_dict = cls.get_elastic_settings_with_user_uploads(
elastic_dict)
idx_type_keys_public = []
idx_type_keys_private = []
# get private idx_keys if auth_public is False and public idx_keys otherwise
idx_keys_public = [
idx_key for (idx_key, idx_values) in elastic_dict.items() if
('auth_public' in idx_values and idx_values['auth_public'] is True)
]
idx_keys_private = [
idx_key for (idx_key, idx_values) in elastic_dict.items()
if ('auth_public' not in idx_values
or idx_values['auth_public'] is False)
]
# get idx_type_keys
for idx_key in idx_keys_public + idx_keys_private:
if 'idx_type' in elastic_dict[idx_key]:
for (idx_type_key, idx_type_values
) in elastic_dict[idx_key]['idx_type'].items():
if 'auth_public' in idx_type_values and idx_type_values[
'auth_public'] is True:
idx_type_keys_public.append(idx_key + '.' +
idx_type_key)
if 'auth_public' not in idx_type_values or idx_type_values[
'auth_public'] is False:
idx_type_keys_private.append(idx_key + '.' +
idx_type_key)
if auth_public:
# make case insensitive
idx_keys_public_icase = [x.upper() for x in idx_keys_public]
idx_type_keys_public_icase = [
x.upper() for x in idx_type_keys_public
]
return idx_keys_public_icase, idx_type_keys_public_icase
else:
# make case insensitive
idx_keys_private_icase = [x.upper() for x in idx_keys_private]
idx_type_keys_private_icase = [
x.upper() for x in idx_type_keys_private
]
return idx_keys_private_icase, idx_type_keys_private_icase
def test_create_idx_type_model_permissions(self):
elastic_dict = elastic_factory.get_elastic_settings_with_user_uploads()
indexKey = 'CP_STATS_UD'
user_upload_dict = list(elastic_dict[indexKey]['idx_type'].keys())
indexName = 'CP_STATS_UD'
if len(user_upload_dict) > 1:
indexType = user_upload_dict[0]
else:
indexType = 'UD-RANDOMTMPTEST'
elastic_dict = ElasticSettings.attrs().get('IDX')
idx_type_dict = {}
idx_type_dict[indexType] = {
'label': 'testlabel',
'type': indexType
}
elastic_dict['CP_STATS_UD']['idx_type'] = idx_type_dict
indexTypeModel = indexKey.lower() + elastic_factory.PERMISSION_MODEL_NAME_TYPE_DELIMITER + indexType.lower() + \
elastic_factory.PERMISSION_MODEL_TYPE_SUFFIX
elastic_factory.create_idx_type_model_permissions(
self.user,
elastic_dict=elastic_dict,
indexKey=indexName,
indexTypeKey=indexType)
# check if model exists
db_models = elastic_factory.get_db_models(
app_label=elastic_factory.PERMISSION_MODEL_APP_NAME, existing=True)
self.assertIn(indexTypeModel, db_models,
' model exists for ' + indexType)
# check if permission exists
content_type = None
try:
content_type = ContentType.objects.get(
model=indexTypeModel.lower(),
app_label=settings.ELASTIC_PERMISSION_MODEL_APP_NAME)
except:
pass
permissions = None
perm_code_name = 'can_read_' + indexTypeModel.lower()
if content_type:
permissions = Permission.objects.filter(content_type=content_type)
self.assertIn(perm_code_name,
[perm.codename for perm in permissions],
'perm present')
# Request new instance of User
user = get_object_or_404(User, pk=self.user.id)
# Permission cache is repopulated from the database
self.assertTrue(
user.has_perm(settings.ELASTIC_PERMISSION_MODEL_APP_NAME +
'.' + perm_code_name))
def get_elastic_settings_with_user_uploads(cls, elastic_dict=None, new_upload_file=None):
'''Get the updated elastic settings with user uploaded idx_types'''
idx_key = 'CP_STATS_UD'
idx = ElasticSettings.idx(idx_key)
''' Check if an index type exists in elastic and later check there is a contenttype/model for the given elastic index type. ''' # @IgnorePep8
elastic_url = ElasticSettings.url()
url = idx + '/_mapping'
response = Search.elastic_request(elastic_url, url, is_post=False)
''' why don't we use Search.get_mapping ? I guess it's not a class method'''
#logger.debug(response.json())
if "error" in response.json():
logger.warn(response.json())
return None
# get idx_types from _mapping
elastic_mapping = json.loads(response.content.decode("utf-8"))
# here if we use aliasing then idx can be different
# this causes problems as it's effectively hardcoded
# this should fix to handle things where aliases are deployed
idx = list(elastic_mapping.keys())[0]
idx_types = list(elastic_mapping[idx]['mappings'].keys())
if elastic_dict is None:
elastic_dict = ElasticSettings.attrs().get('IDX')
idx_type_dict = {}
existing_ct = [ct.name for ct in ContentType.objects.filter(app_label=cls.PERMISSION_MODEL_APP_NAME)]
for idx_type in idx_types:
idx_type_with_suffix = idx_type + cls.PERMISSION_MODEL_TYPE_SUFFIX
for ct in existing_ct:
if ct.endswith(idx_type_with_suffix):
meta_url = idx + '/' + idx_type + '/_meta/_source'
meta_response = Search.elastic_request(elastic_url, meta_url, is_post=False)
try:
elastic_meta = json.loads(meta_response.content.decode("utf-8"))
label = elastic_meta['label']
except:
label = "UD-" + idx_type
idx_type_dict['UD-' + idx_type.upper()] = {'label': label, 'type': idx_type}
if new_upload_file is not None:
idx_type = new_upload_file
label = "UD-" + idx_type
idx_type_dict['UD-' + idx_type.upper()] = {'label': label, 'type': idx_type}
elastic_dict['CP_STATS_UD']['idx_type'] = idx_type_dict
return elastic_dict
def test_create_idx_type_model_permissions(self):
elastic_dict = elastic_factory.get_elastic_settings_with_user_uploads()
indexKey = 'CP_STATS_UD'
user_upload_dict = list(elastic_dict[indexKey]['idx_type'].keys())
indexName = 'CP_STATS_UD'
if len(user_upload_dict) > 1:
indexType = user_upload_dict[0]
else:
indexType = 'UD-RANDOMTMPTEST'
elastic_dict = ElasticSettings.attrs().get('IDX')
idx_type_dict = {}
idx_type_dict[indexType] = {'label': 'testlabel', 'type': indexType}
elastic_dict['CP_STATS_UD']['idx_type'] = idx_type_dict
indexTypeModel = indexKey.lower() + elastic_factory.PERMISSION_MODEL_NAME_TYPE_DELIMITER + indexType.lower() + \
elastic_factory.PERMISSION_MODEL_TYPE_SUFFIX
elastic_factory.create_idx_type_model_permissions(self.user, elastic_dict=elastic_dict,
indexKey=indexName,
indexTypeKey=indexType)
# check if model exists
db_models = elastic_factory.get_db_models(app_label=elastic_factory.PERMISSION_MODEL_APP_NAME, existing=True)
self.assertIn(indexTypeModel, db_models,
' model exists for ' + indexType)
# check if permission exists
content_type = None
try:
content_type = ContentType.objects.get(model=indexTypeModel.lower(),
app_label=settings.ELASTIC_PERMISSION_MODEL_APP_NAME)
except:
pass
permissions = None
perm_code_name = 'can_read_' + indexTypeModel.lower()
if content_type:
permissions = Permission.objects.filter(content_type=content_type)
self.assertIn(perm_code_name, [perm.codename for perm in permissions], 'perm present')
# Request new instance of User
user = get_object_or_404(User, pk=self.user.id)
# Permission cache is repopulated from the database
self.assertTrue(user.has_perm(settings.ELASTIC_PERMISSION_MODEL_APP_NAME + '.' + perm_code_name))
def get_idx_and_idx_type_keys(cls, elastic_dict=None, auth_public=False):
'''
Returns the idx and idx type keys
Use the auth_public flag to restrict to either public or private idx and idx_types
'''
if elastic_dict is None:
elastic_dict = ElasticSettings.attrs().get('IDX')
connection = connections[settings.AUTH_DB]
if settings.INCLUDE_USER_UPLOADS is True:
if "django_content_type" in connection.introspection.table_names():
elastic_dict = cls.get_elastic_settings_with_user_uploads(elastic_dict)
idx_type_keys_public = []
idx_type_keys_private = []
# get private idx_keys if auth_public is False and public idx_keys otherwise
idx_keys_public = [idx_key for (idx_key, idx_values) in elastic_dict.items()
if('auth_public' in idx_values and idx_values['auth_public'] is True)]
idx_keys_private = [idx_key for (idx_key, idx_values) in elastic_dict.items()
if('auth_public' not in idx_values or idx_values['auth_public'] is False)]
# get idx_type_keys
for idx_key in idx_keys_public + idx_keys_private:
if 'idx_type' in elastic_dict[idx_key]:
for (idx_type_key, idx_type_values) in elastic_dict[idx_key]['idx_type'].items():
if 'auth_public' in idx_type_values and idx_type_values['auth_public'] is True:
idx_type_keys_public.append(idx_key + '.' + idx_type_key)
if 'auth_public' not in idx_type_values or idx_type_values['auth_public'] is False:
idx_type_keys_private.append(idx_key + '.' + idx_type_key)
if auth_public:
# make case insensitive
idx_keys_public_icase = [x.upper() for x in idx_keys_public]
idx_type_keys_public_icase = [x.upper() for x in idx_type_keys_public]
return idx_keys_public_icase, idx_type_keys_public_icase
else:
# make case insensitive
idx_keys_private_icase = [x.upper() for x in idx_keys_private]
idx_type_keys_private_icase = [x.upper() for x in idx_type_keys_private]
return idx_keys_private_icase, idx_type_keys_private_icase
def get_elastic_settings_with_user_uploads(cls, elastic_dict=None):
'''Get the updated elastic settings with user uploaded idx_types'''
idx_key = 'CP_STATS_UD'
idx = ElasticSettings.idx(idx_key)
''' Check if an index exists. '''
elastic_url = ElasticSettings.url()
url = idx + '/_mapping'
response = Search.elastic_request(elastic_url, url, is_post=False)
if "error" in response.json():
logger.warn(response.json())
return None
# get idx_types from _mapping
elastic_mapping = json.loads(response.content.decode("utf-8"))
idx_types = list(elastic_mapping[idx]['mappings'].keys())
if elastic_dict is None:
elastic_dict = ElasticSettings.attrs().get('IDX')
idx_type_dict = {}
for idx_type in idx_types:
meta_url = idx + '/' + idx_type + '/_meta/_source'
meta_response = Search.elastic_request(elastic_url, meta_url, is_post=False)
try:
elastic_meta = json.loads(meta_response.content.decode("utf-8"))
label = elastic_meta['label']
except:
label = "UD-" + idx_type
idx_type_dict['UD-' + idx_type.upper()] = {'label': label, 'type': idx_type}
elastic_dict['CP_STATS_UD']['idx_type'] = idx_type_dict
return elastic_dict
def get_elastic_settings_with_user_uploads(cls,
elastic_dict=None,
new_upload_file=None):
'''Get the updated elastic settings with user uploaded idx_types'''
idx_key = 'CP_STATS_UD'
idx = ElasticSettings.idx(idx_key)
''' Check if an index type exists in elastic and later check there is a contenttype/model for the given elastic index type. ''' # @IgnorePep8
elastic_url = ElasticSettings.url()
url = idx + '/_mapping'
response = Search.elastic_request(elastic_url, url, is_post=False)
''' why don't we use Search.get_mapping ? I guess it's not a class method'''
#logger.debug(response.json())
if "error" in response.json():
logger.warn(response.json())
return None
# get idx_types from _mapping
elastic_mapping = json.loads(response.content.decode("utf-8"))
# here if we use aliasing then idx can be different
# this causes problems as it's effectively hardcoded
# this should fix to handle things where aliases are deployed
idx = list(elastic_mapping.keys())[0]
idx_types = list(elastic_mapping[idx]['mappings'].keys())
if elastic_dict is None:
elastic_dict = ElasticSettings.attrs().get('IDX')
idx_type_dict = {}
existing_ct = [
ct.name for ct in ContentType.objects.filter(
app_label=cls.PERMISSION_MODEL_APP_NAME)
]
for idx_type in idx_types:
idx_type_with_suffix = idx_type + cls.PERMISSION_MODEL_TYPE_SUFFIX
for ct in existing_ct:
if ct.endswith(idx_type_with_suffix):
meta_url = idx + '/' + idx_type + '/_meta/_source'
meta_response = Search.elastic_request(elastic_url,
meta_url,
is_post=False)
try:
elastic_meta = json.loads(
meta_response.content.decode("utf-8"))
label = elastic_meta['label']
except:
label = "UD-" + idx_type
idx_type_dict['UD-' + idx_type.upper()] = {
'label': label,
'type': idx_type
}
if new_upload_file is not None:
idx_type = new_upload_file
label = "UD-" + idx_type
idx_type_dict['UD-' + idx_type.upper()] = {
'label': label,
'type': idx_type
}
elastic_dict['CP_STATS_UD']['idx_type'] = idx_type_dict
return elastic_dict