def test_create_idx_type_model_permissions(self): elastic_settings_before = ElasticSettings.attrs().get('IDX') user_types_before = elastic_settings_before['CP_STATS_UD']['idx_type'] self.assertEqual({}, user_types_before, 'CP_STATS_UD idx_type is empty') idx = "cp:hg19_userdata_bed" new_upload_file = "tmp_newly_uploaded_file" idx_type = new_upload_file os.system( "curl -XPUT " + ElasticSettings.url() + "/" + idx + "/_mapping/" + idx_type + " -d '{\"" + idx_type + "\":{ \"properties\" : {\"message\" : {\"type\" : \"string\", \"store\" : true } } }}'" ) os.system("curl -XPUT " + ElasticSettings.url() + "/" + idx + "/" + idx_type + "/_meta -d '{\"label\": \"" + new_upload_file + "\", \"owner\": \"" + self.user.username + "\", \"uploaded\": \"" + str(timezone.now()) + "\"}'") elastic_settings_after = elastic_factory.create_idx_type_model_permissions( self.user, indexKey='CP_STATS_UD', indexTypeKey='UD-' + new_upload_file.upper(), # @IgnorePep8 new_upload_file="tmp_newly_uploaded_file") # @IgnorePep8 # elastic_settings_after = elastic_factory.get_elastic_settings_with_user_uploads(elastic_settings_before) user_types_after = elastic_settings_after['CP_STATS_UD']['idx_type'] self.assertTrue(len(user_types_after) > 0, "Has user idx_types ") self.assertTrue('UD-TMP_NEWLY_UPLOADED_FILE' in user_types_after) self.assertEqual( user_types_after['UD-TMP_NEWLY_UPLOADED_FILE']['type'], 'tmp_newly_uploaded_file')
def test_create_idx_type_model_permissions(self): elastic_settings_before = ElasticSettings.attrs().get('IDX') user_types_before = elastic_settings_before['CP_STATS_UD']['idx_type'] self.assertEqual({}, user_types_before, 'CP_STATS_UD idx_type is empty') idx = "cp:hg19_userdata_bed" new_upload_file = "tmp_newly_uploaded_file" idx_type = new_upload_file os.system("curl -XPUT "+ElasticSettings.url()+"/"+idx+"/_mapping/"+idx_type+" -d '{\"" + idx_type + "\":{ \"properties\" : {\"message\" : {\"type\" : \"string\", \"store\" : true } } }}'") os.system("curl -XPUT "+ElasticSettings.url()+"/"+idx+"/"+idx_type+"/_meta -d '{\"label\": \"" + new_upload_file + "\", \"owner\": \""+self.user.username+"\", \"uploaded\": \"" + str(timezone.now())+"\"}'") elastic_settings_after = elastic_factory.create_idx_type_model_permissions(self.user, indexKey='CP_STATS_UD', indexTypeKey='UD-'+new_upload_file.upper(), # @IgnorePep8 new_upload_file="tmp_newly_uploaded_file") # @IgnorePep8 # elastic_settings_after = elastic_factory.get_elastic_settings_with_user_uploads(elastic_settings_before) user_types_after = elastic_settings_after['CP_STATS_UD']['idx_type'] self.assertTrue(len(user_types_after) > 0, "Has user idx_types ") self.assertTrue('UD-TMP_NEWLY_UPLOADED_FILE' in user_types_after) self.assertEqual(user_types_after['UD-TMP_NEWLY_UPLOADED_FILE']['type'], 'tmp_newly_uploaded_file')
def get_idx_and_idx_type_keys(cls, elastic_dict=None, auth_public=False): ''' Returns the idx and idx type keys Use the auth_public flag to restrict to either public or private idx and idx_types ''' if elastic_dict is None: elastic_dict = ElasticSettings.attrs().get('IDX') connection = connections[settings.AUTH_DB] if settings.INCLUDE_USER_UPLOADS is True: if "django_content_type" in connection.introspection.table_names( ): elastic_dict = cls.get_elastic_settings_with_user_uploads( elastic_dict) idx_type_keys_public = [] idx_type_keys_private = [] # get private idx_keys if auth_public is False and public idx_keys otherwise idx_keys_public = [ idx_key for (idx_key, idx_values) in elastic_dict.items() if ('auth_public' in idx_values and idx_values['auth_public'] is True) ] idx_keys_private = [ idx_key for (idx_key, idx_values) in elastic_dict.items() if ('auth_public' not in idx_values or idx_values['auth_public'] is False) ] # get idx_type_keys for idx_key in idx_keys_public + idx_keys_private: if 'idx_type' in elastic_dict[idx_key]: for (idx_type_key, idx_type_values ) in elastic_dict[idx_key]['idx_type'].items(): if 'auth_public' in idx_type_values and idx_type_values[ 'auth_public'] is True: idx_type_keys_public.append(idx_key + '.' + idx_type_key) if 'auth_public' not in idx_type_values or idx_type_values[ 'auth_public'] is False: idx_type_keys_private.append(idx_key + '.' + idx_type_key) if auth_public: # make case insensitive idx_keys_public_icase = [x.upper() for x in idx_keys_public] idx_type_keys_public_icase = [ x.upper() for x in idx_type_keys_public ] return idx_keys_public_icase, idx_type_keys_public_icase else: # make case insensitive idx_keys_private_icase = [x.upper() for x in idx_keys_private] idx_type_keys_private_icase = [ x.upper() for x in idx_type_keys_private ] return idx_keys_private_icase, idx_type_keys_private_icase
def test_create_idx_type_model_permissions(self): elastic_dict = elastic_factory.get_elastic_settings_with_user_uploads() indexKey = 'CP_STATS_UD' user_upload_dict = list(elastic_dict[indexKey]['idx_type'].keys()) indexName = 'CP_STATS_UD' if len(user_upload_dict) > 1: indexType = user_upload_dict[0] else: indexType = 'UD-RANDOMTMPTEST' elastic_dict = ElasticSettings.attrs().get('IDX') idx_type_dict = {} idx_type_dict[indexType] = { 'label': 'testlabel', 'type': indexType } elastic_dict['CP_STATS_UD']['idx_type'] = idx_type_dict indexTypeModel = indexKey.lower() + elastic_factory.PERMISSION_MODEL_NAME_TYPE_DELIMITER + indexType.lower() + \ elastic_factory.PERMISSION_MODEL_TYPE_SUFFIX elastic_factory.create_idx_type_model_permissions( self.user, elastic_dict=elastic_dict, indexKey=indexName, indexTypeKey=indexType) # check if model exists db_models = elastic_factory.get_db_models( app_label=elastic_factory.PERMISSION_MODEL_APP_NAME, existing=True) self.assertIn(indexTypeModel, db_models, ' model exists for ' + indexType) # check if permission exists content_type = None try: content_type = ContentType.objects.get( model=indexTypeModel.lower(), app_label=settings.ELASTIC_PERMISSION_MODEL_APP_NAME) except: pass permissions = None perm_code_name = 'can_read_' + indexTypeModel.lower() if content_type: permissions = Permission.objects.filter(content_type=content_type) self.assertIn(perm_code_name, [perm.codename for perm in permissions], 'perm present') # Request new instance of User user = get_object_or_404(User, pk=self.user.id) # Permission cache is repopulated from the database self.assertTrue( user.has_perm(settings.ELASTIC_PERMISSION_MODEL_APP_NAME + '.' + perm_code_name))
def get_elastic_settings_with_user_uploads(cls, elastic_dict=None, new_upload_file=None): '''Get the updated elastic settings with user uploaded idx_types''' idx_key = 'CP_STATS_UD' idx = ElasticSettings.idx(idx_key) ''' Check if an index type exists in elastic and later check there is a contenttype/model for the given elastic index type. ''' # @IgnorePep8 elastic_url = ElasticSettings.url() url = idx + '/_mapping' response = Search.elastic_request(elastic_url, url, is_post=False) ''' why don't we use Search.get_mapping ? I guess it's not a class method''' #logger.debug(response.json()) if "error" in response.json(): logger.warn(response.json()) return None # get idx_types from _mapping elastic_mapping = json.loads(response.content.decode("utf-8")) # here if we use aliasing then idx can be different # this causes problems as it's effectively hardcoded # this should fix to handle things where aliases are deployed idx = list(elastic_mapping.keys())[0] idx_types = list(elastic_mapping[idx]['mappings'].keys()) if elastic_dict is None: elastic_dict = ElasticSettings.attrs().get('IDX') idx_type_dict = {} existing_ct = [ct.name for ct in ContentType.objects.filter(app_label=cls.PERMISSION_MODEL_APP_NAME)] for idx_type in idx_types: idx_type_with_suffix = idx_type + cls.PERMISSION_MODEL_TYPE_SUFFIX for ct in existing_ct: if ct.endswith(idx_type_with_suffix): meta_url = idx + '/' + idx_type + '/_meta/_source' meta_response = Search.elastic_request(elastic_url, meta_url, is_post=False) try: elastic_meta = json.loads(meta_response.content.decode("utf-8")) label = elastic_meta['label'] except: label = "UD-" + idx_type idx_type_dict['UD-' + idx_type.upper()] = {'label': label, 'type': idx_type} if new_upload_file is not None: idx_type = new_upload_file label = "UD-" + idx_type idx_type_dict['UD-' + idx_type.upper()] = {'label': label, 'type': idx_type} elastic_dict['CP_STATS_UD']['idx_type'] = idx_type_dict return elastic_dict
def test_create_idx_type_model_permissions(self): elastic_dict = elastic_factory.get_elastic_settings_with_user_uploads() indexKey = 'CP_STATS_UD' user_upload_dict = list(elastic_dict[indexKey]['idx_type'].keys()) indexName = 'CP_STATS_UD' if len(user_upload_dict) > 1: indexType = user_upload_dict[0] else: indexType = 'UD-RANDOMTMPTEST' elastic_dict = ElasticSettings.attrs().get('IDX') idx_type_dict = {} idx_type_dict[indexType] = {'label': 'testlabel', 'type': indexType} elastic_dict['CP_STATS_UD']['idx_type'] = idx_type_dict indexTypeModel = indexKey.lower() + elastic_factory.PERMISSION_MODEL_NAME_TYPE_DELIMITER + indexType.lower() + \ elastic_factory.PERMISSION_MODEL_TYPE_SUFFIX elastic_factory.create_idx_type_model_permissions(self.user, elastic_dict=elastic_dict, indexKey=indexName, indexTypeKey=indexType) # check if model exists db_models = elastic_factory.get_db_models(app_label=elastic_factory.PERMISSION_MODEL_APP_NAME, existing=True) self.assertIn(indexTypeModel, db_models, ' model exists for ' + indexType) # check if permission exists content_type = None try: content_type = ContentType.objects.get(model=indexTypeModel.lower(), app_label=settings.ELASTIC_PERMISSION_MODEL_APP_NAME) except: pass permissions = None perm_code_name = 'can_read_' + indexTypeModel.lower() if content_type: permissions = Permission.objects.filter(content_type=content_type) self.assertIn(perm_code_name, [perm.codename for perm in permissions], 'perm present') # Request new instance of User user = get_object_or_404(User, pk=self.user.id) # Permission cache is repopulated from the database self.assertTrue(user.has_perm(settings.ELASTIC_PERMISSION_MODEL_APP_NAME + '.' + perm_code_name))
def get_idx_and_idx_type_keys(cls, elastic_dict=None, auth_public=False): ''' Returns the idx and idx type keys Use the auth_public flag to restrict to either public or private idx and idx_types ''' if elastic_dict is None: elastic_dict = ElasticSettings.attrs().get('IDX') connection = connections[settings.AUTH_DB] if settings.INCLUDE_USER_UPLOADS is True: if "django_content_type" in connection.introspection.table_names(): elastic_dict = cls.get_elastic_settings_with_user_uploads(elastic_dict) idx_type_keys_public = [] idx_type_keys_private = [] # get private idx_keys if auth_public is False and public idx_keys otherwise idx_keys_public = [idx_key for (idx_key, idx_values) in elastic_dict.items() if('auth_public' in idx_values and idx_values['auth_public'] is True)] idx_keys_private = [idx_key for (idx_key, idx_values) in elastic_dict.items() if('auth_public' not in idx_values or idx_values['auth_public'] is False)] # get idx_type_keys for idx_key in idx_keys_public + idx_keys_private: if 'idx_type' in elastic_dict[idx_key]: for (idx_type_key, idx_type_values) in elastic_dict[idx_key]['idx_type'].items(): if 'auth_public' in idx_type_values and idx_type_values['auth_public'] is True: idx_type_keys_public.append(idx_key + '.' + idx_type_key) if 'auth_public' not in idx_type_values or idx_type_values['auth_public'] is False: idx_type_keys_private.append(idx_key + '.' + idx_type_key) if auth_public: # make case insensitive idx_keys_public_icase = [x.upper() for x in idx_keys_public] idx_type_keys_public_icase = [x.upper() for x in idx_type_keys_public] return idx_keys_public_icase, idx_type_keys_public_icase else: # make case insensitive idx_keys_private_icase = [x.upper() for x in idx_keys_private] idx_type_keys_private_icase = [x.upper() for x in idx_type_keys_private] return idx_keys_private_icase, idx_type_keys_private_icase
def get_elastic_settings_with_user_uploads(cls, elastic_dict=None): '''Get the updated elastic settings with user uploaded idx_types''' idx_key = 'CP_STATS_UD' idx = ElasticSettings.idx(idx_key) ''' Check if an index exists. ''' elastic_url = ElasticSettings.url() url = idx + '/_mapping' response = Search.elastic_request(elastic_url, url, is_post=False) if "error" in response.json(): logger.warn(response.json()) return None # get idx_types from _mapping elastic_mapping = json.loads(response.content.decode("utf-8")) idx_types = list(elastic_mapping[idx]['mappings'].keys()) if elastic_dict is None: elastic_dict = ElasticSettings.attrs().get('IDX') idx_type_dict = {} for idx_type in idx_types: meta_url = idx + '/' + idx_type + '/_meta/_source' meta_response = Search.elastic_request(elastic_url, meta_url, is_post=False) try: elastic_meta = json.loads(meta_response.content.decode("utf-8")) label = elastic_meta['label'] except: label = "UD-" + idx_type idx_type_dict['UD-' + idx_type.upper()] = {'label': label, 'type': idx_type} elastic_dict['CP_STATS_UD']['idx_type'] = idx_type_dict return elastic_dict
def get_elastic_settings_with_user_uploads(cls, elastic_dict=None, new_upload_file=None): '''Get the updated elastic settings with user uploaded idx_types''' idx_key = 'CP_STATS_UD' idx = ElasticSettings.idx(idx_key) ''' Check if an index type exists in elastic and later check there is a contenttype/model for the given elastic index type. ''' # @IgnorePep8 elastic_url = ElasticSettings.url() url = idx + '/_mapping' response = Search.elastic_request(elastic_url, url, is_post=False) ''' why don't we use Search.get_mapping ? I guess it's not a class method''' #logger.debug(response.json()) if "error" in response.json(): logger.warn(response.json()) return None # get idx_types from _mapping elastic_mapping = json.loads(response.content.decode("utf-8")) # here if we use aliasing then idx can be different # this causes problems as it's effectively hardcoded # this should fix to handle things where aliases are deployed idx = list(elastic_mapping.keys())[0] idx_types = list(elastic_mapping[idx]['mappings'].keys()) if elastic_dict is None: elastic_dict = ElasticSettings.attrs().get('IDX') idx_type_dict = {} existing_ct = [ ct.name for ct in ContentType.objects.filter( app_label=cls.PERMISSION_MODEL_APP_NAME) ] for idx_type in idx_types: idx_type_with_suffix = idx_type + cls.PERMISSION_MODEL_TYPE_SUFFIX for ct in existing_ct: if ct.endswith(idx_type_with_suffix): meta_url = idx + '/' + idx_type + '/_meta/_source' meta_response = Search.elastic_request(elastic_url, meta_url, is_post=False) try: elastic_meta = json.loads( meta_response.content.decode("utf-8")) label = elastic_meta['label'] except: label = "UD-" + idx_type idx_type_dict['UD-' + idx_type.upper()] = { 'label': label, 'type': idx_type } if new_upload_file is not None: idx_type = new_upload_file label = "UD-" + idx_type idx_type_dict['UD-' + idx_type.upper()] = { 'label': label, 'type': idx_type } elastic_dict['CP_STATS_UD']['idx_type'] = idx_type_dict return elastic_dict