def to_representation(self, data):
ret = super(ListingSerializer, self).to_representation(data)
anonymize_identifiable_data = system_anonymize_identifiable_data(self.context['request'].user.username)
request_user = generic_model_access.get_profile(self.context['request'].user) # TODO: Get the profile from view request instead of getting it from db again
if anonymize_identifiable_data:
ret['contacts'] = []
check_failed = []
# owners
if 'owners' in ret and not anonymize_identifiable_data:
for owner in ret['owners']:
user_dict = owner.get('user')
user_username = None if user_dict is None else user_dict.get('username')
# if not user_username:
# raise serializers.ValidationError('Owner field requires correct format')
owner_profile = generic_model_access.get_profile(user_username)
# if not owner_profile:
# raise serializers.ValidationError('Owner Profile not found')
# Don't allow user to select a security marking that is above
# their own access level\
try:
if system_has_access_control(owner_profile.user.username, ret.get('security_marking')) is False:
check_failed.append(owner_profile.user.username)
# raise serializers.ValidationError(owner_profile.user.username + 'User certificate is invalid')
except Exception:
check_failed.append(owner_profile.user.username)
ret['cert_issues'] = check_failed
ret['feedback'] = self._feedback(request_user, data)
ret['is_bookmarked'] = self._is_bookmarked(request_user, data)
return ret
def for_user(self, username):
objects = super(AccessControlRecommendationFeedbackManager,
self).get_queryset()
profile_instance = Profile.objects.get(user__username=username)
# filter out private listings
exclude_orgs = get_user_excluded_orgs(profile_instance)
objects = objects.filter(
target_profile=profile_instance,
target_listing__is_enabled=True,
target_listing__approval_status=Listing.APPROVED,
target_listing__is_deleted=False)
objects = objects.exclude(target_listing__is_private=True,
target_listing__agency__in=exclude_orgs)
# Filter out listings by user's access level
ids_to_exclude = []
for recommend_feedback_obj in objects:
if not recommend_feedback_obj.target_listing.security_marking:
logger.debug('Listing {0!s} has no security_marking'.format(
recommend_feedback_obj.target_listing.title))
if not system_has_access_control(
username,
recommend_feedback_obj.target_listing.security_marking):
ids_to_exclude.append(recommend_feedback_obj.target_listing.id)
objects = objects.exclude(target_listing__pk__in=ids_to_exclude)
return objects
def suggest(request_username, search_param_parser):
"""
Suggest
It must respects restrictions
- Private apps (apps only from user's agency)
- User's max_classification_level
Args:
request_username(string)
search_param_parser(SearchParamParser): Parsed Request Search Object
Returns:
listing titles in a list
"""
# Create ES client
es_client = elasticsearch_factory.get_client()
elasticsearch_factory.check_elasticsearch()
if search_param_parser.search_string is None:
return []
user_exclude_orgs = get_user_exclude_orgs(request_username)
# Override Limit - Only 15 results should come if limit was not set
if search_param_parser.limit_set is False:
search_param_parser.limit = constants.ES_SUGGEST_LIMIT
search_query = elasticsearch_util.make_search_query_obj(
search_param_parser, exclude_agencies=user_exclude_orgs)
# Only Retrieve ['title', 'security_marking', 'id'] fields from Elasticsearch for suggestions
search_query['_source'] = ['title', 'security_marking', 'id']
# print(json.dumps(search_query, indent=4)) # Print statement for debugging output
res = es_client.search(index=settings.ES_INDEX_NAME, body=search_query)
hits = res.get('hits', {}).get('hits', None)
if not hits:
return []
hit_titles = []
for hit in hits:
source = hit.get('_source')
exclude_bool = False
if not source.get('security_marking'):
exclude_bool = True
logger.debug('Listing {0!s} has no security_marking'.format(
source.get('title')))
if not system_has_access_control(request_username,
source.get('security_marking')):
exclude_bool = True
if exclude_bool is False:
temp = {'title': source['title'], 'id': source['id']}
hit_titles.append(temp)
return hit_titles
def process_next_start(self):
"""
execute security_marking check on each listing
"""
while True:
listing = self.starts.next()
if not listing['security_marking']:
logger.debug('Listing {0!s} has no security_marking'.format(
listing['title']))
else:
if self.featured:
if listing['is_featured'] is True:
if system_has_access_control(
self.username, listing['security_marking']):
return listing
else:
if system_has_access_control(self.username,
listing['security_marking']):
return listing
def validate_security_marking(self, value):
# don't allow user to select a security marking that is above
# their own access level
username = self.context['request'].user.username
if value:
if not system_has_access_control(username, value):
raise serializers.ValidationError(
'Security marking too high for current user')
return value
def process_next_start(self):
"""
execute security_marking check on each listing
"""
while True:
listing = self.starts.next()
if not listing.security_marking:
logger.debug('Listing {0!s} has no security_marking'.format(
listing.title))
if system_has_access_control(self.username,
listing.security_marking):
return listing
def validate_security_marking(self, value):
# don't allow user to select a security marking that is above
# their own access level
profile = generic_model_access.get_profile(
self.context['request'].user.username)
if value:
if not system_has_access_control(profile.user.username, value):
raise serializers.ValidationError(
'Security marking too high for current user')
else:
raise serializers.ValidationError('Security marking is required')
return value
def for_user(self, username):
objects = super(AccessControlListingManager, self).get_queryset()
profile_instance = Profile.objects.get(user__username=username)
# filter out private listings
exclude_orgs = get_user_excluded_orgs(profile_instance)
objects = objects.exclude(is_private=True, agency__in=exclude_orgs)
objects = self.apply_select_related(objects)
# Filter out listings by user's access level
ids_to_exclude = []
for i in objects:
if not i.security_marking:
logger.debug('Listing {0!s} has no security_marking'.format(
i.title))
if not system_has_access_control(username, i.security_marking):
ids_to_exclude.append(i.id)
objects = objects.exclude(pk__in=ids_to_exclude)
return objects
def for_user(self, username):
"""
This method causes:
sqlite3.OperationalError: too many SQL variables
To fix this error, post filtering in memory needs to happen
Example Code:
serializer = serializers.ImageSerializer(queryset, many=True, context={'request': request})
serializer_iterator = recommend_utils.ListIterator(serializer.data)
pipeline_list = [pipes.ListingDictPostSecurityMarkingCheckPipe(request.user.username)]
recommended_listings = pipeline.Pipeline(serializer_iterator, pipeline_list).to_list()
"""
objects = super(AccessControlImageManager, self).get_queryset()
# filter out listings by user's access level
images_to_exclude = []
for i in objects:
if not system_has_access_control(username, i.security_marking):
images_to_exclude.append(i.id)
objects = objects.exclude(id__in=images_to_exclude)
return objects
def search(request_username, search_param_parser):
"""
Filter Listings
Too many variations to cache results
Users shall be able to search for listings'
- title
- description
- description_short
- tags__name
Filter by
- category
- agency
- listing types
- exportable
Users shall only see what they are authorized to see
'is_private': false,
'approval_status': 'APPROVED',
'is_deleted': false,
'is_enabled': true,
'security_marking': 'UNCLASSIFIED',
Sorted by Relevance
'avg_rate': 0,
'total_votes': 0,
'total_rate5': 0,
'total_rate4': 0,
'total_rate3': 0,
'total_rate2': 0,
'total_rate1': 0,
'total_reviews': 0,
'is_featured': true,
It must respects restrictions
- Private apps (apps only from user's agency)
- User's max_classification_level
Args:
username(str): username
search_param_parser(SearchParamParser): parameters
"""
elasticsearch_factory.check_elasticsearch()
# Create ES client
es_client = elasticsearch_factory.get_client()
user_exclude_orgs = get_user_exclude_orgs(request_username)
search_query = elasticsearch_util.make_search_query_obj(
search_param_parser, exclude_agencies=user_exclude_orgs)
# print(json.dumps(search_query, indent=4))
res = es_client.search(index=settings.ES_INDEX_NAME, body=search_query)
hits = res.get('hits', {})
inner_hits = hits.get('hits', None)
if not hits:
return []
hit_titles = []
excluded_count = 0
for current_innter_hit in inner_hits:
source = current_innter_hit.get('_source')
source['_score'] = current_innter_hit.get('_score')
# Add URLs to icons
image_keys_to_add_url = [
'large_icon', 'small_icon', 'banner_icon', 'large_banner_icon'
]
for image_key in image_keys_to_add_url:
if source.get(image_key) is not None:
if search_param_parser.base_url:
source[image_key]['url'] = '{!s}/api/image/{!s}/'.format(
search_param_parser.base_url, source[image_key]['id'])
else:
source[image_key]['url'] = '/api/image/{!s}/'.format(
source[image_key]['id'])
exclude_bool = False
if not source.get('security_marking'):
exclude_bool = True
logger.debug('Listing {0!s} has no security_marking'.format(
source.get('title')))
if not system_has_access_control(request_username,
source.get('security_marking')):
exclude_bool = True
if exclude_bool is False:
hit_titles.append(source)
else:
excluded_count = excluded_count + 1
# Total Records in Elasticsearch
final_count = hits.get('total')
# Total Records minus what the user does not have access to see, this count should never be below zero
# TODO: Figure out smarter logic for excluded_count compensation (rivera 11/14/2016)
final_count_with_excluded = final_count - excluded_count
final_results = {'count': final_count_with_excluded, 'results': hit_titles}
final_results['previous'] = None
final_results['next'] = None
# if final_count_with_excluded < 0 then previous and next should be None
if final_count_with_excluded < 0:
return final_results
previous_offset_prediction = search_param_parser.offset - search_param_parser.limit
next_offset_prediction = search_param_parser.offset + search_param_parser.limit
final_results['next_offset_prediction'] = next_offset_prediction
# Previous URL - previous_offset_prediction is less than zero, previous should be None
if previous_offset_prediction >= 0:
final_results['previous'] = generate_link(search_param_parser,
previous_offset_prediction)
# Next URL
if next_offset_prediction <= final_count_with_excluded:
final_results['next'] = generate_link(search_param_parser,
next_offset_prediction)
return final_results
def validate_listing(serializer_instance, data):
# TODO Put in listing model_access.py
access_control_instance = plugin_manager.get_system_access_control_plugin()
profile = generic_model_access.get_profile(serializer_instance.context['request'].user.username)
# This checks to see if value exist as a key and value is not None
if not data.get('title'):
raise serializers.ValidationError('Title is required')
if 'security_marking' not in data:
raise serializers.ValidationError('Security Marking is Required')
# Assign a default security_marking level if none is provided
if not data.get('security_marking'):
data['security_marking'] = constants.DEFAULT_SECURITY_MARKING
if not access_control_instance.validate_marking(data['security_marking']):
raise serializers.ValidationError('Security Marking Format is Invalid')
# Don't allow user to select a security marking that is above
# their own access level
if not system_has_access_control(profile.user.username, data.get('security_marking')):
raise serializers.ValidationError('Security marking too high for current user')
# Don't allow 2nd-party user to be an submit/edit a listing
if system_anonymize_identifiable_data(profile.user.username):
raise serializers.ValidationError('Permissions are invalid for current profile')
# TODO: errors.PermissionDenied instead of serializers.ValidationError
data['description'] = data.get('description')
data['launch_url'] = data.get('launch_url')
data['version_name'] = data.get('version_name')
data['unique_name'] = data.get('unique_name')
data['what_is_new'] = data.get('what_is_new')
data['description_short'] = data.get('description_short')
data['usage_requirements'] = data.get('usage_requirements')
data['system_requirements'] = data.get('system_requirements')
data['is_private'] = data.get('is_private', False)
data['is_exportable'] = data.get('is_exportable', False)
data['security_marking'] = data.get('security_marking')
# only checked on update, not create
data['is_enabled'] = data.get('is_enabled', False)
data['is_508_compliant'] = data.get('is_508_compliant', False)
data['is_featured'] = data.get('is_featured', False)
data['approval_status'] = data.get('approval_status')
# Agency
agency_title = None if data.get('agency') is None else data.get('agency', {}).get('title')
if agency_title:
data['agency'] = agency_model_access.get_agency_by_title(agency_title)
if data['agency'] is None:
raise serializers.ValidationError('Invalid Agency')
else:
data['agency'] = profile.organizations.all()[0]
# Listing Type
type_title = None if data.get('listing_type') is None else data.get('listing_type', {}).get('title')
if type_title:
data['listing_type'] = model_access.get_listing_type_by_title(type_title)
else:
data['listing_type'] = None
# Images
image_keys = ['small_icon', 'large_icon', 'banner_icon', 'large_banner_icon']
for image_key in image_keys:
current_image_value = data.get(image_key)
if current_image_value:
if 'id' not in current_image_value:
raise serializers.ValidationError('Image({!s}) requires a {!s}'.format(image_key, 'id'))
if current_image_value.get('security_marking') is None:
current_image_value['security_marking'] = constants.DEFAULT_SECURITY_MARKING
if not access_control_instance.validate_marking(current_image_value['security_marking']):
raise errors.InvalidInput('{!s} Security Marking is invalid'.format(image_key))
else:
data[image_key] = None
# Screenshot
screenshots = data.get('screenshots')
if screenshots is not None:
screenshots_out = []
image_require_fields = ['id']
for screenshot_set in screenshots:
if ('small_image' not in screenshot_set or
'large_image' not in screenshot_set):
raise serializers.ValidationError(
'Screenshot Set requires {0!s} fields'.format('small_image, large_icon'))
screenshot_small_image = screenshot_set.get('small_image')
screenshot_large_image = screenshot_set.get('large_image')
for field in image_require_fields:
if field not in screenshot_small_image:
raise serializers.ValidationError('Screenshot Small Image requires a {0!s}'.format(field))
for field in image_require_fields:
if field not in screenshot_large_image:
raise serializers.ValidationError('Screenshot Large Image requires a {0!s}'.format(field))
if not screenshot_small_image.get('security_marking'):
screenshot_small_image['security_marking'] = constants.DEFAULT_SECURITY_MARKING
if not access_control_instance.validate_marking(screenshot_small_image['security_marking']):
raise errors.InvalidInput('Security Marking is invalid')
if not screenshot_large_image.get('security_marking'):
screenshot_large_image['security_marking'] = constants.DEFAULT_SECURITY_MARKING
if not access_control_instance.validate_marking(screenshot_large_image['security_marking']):
raise errors.InvalidInput('Security Marking is invalid')
screenshots_out.append(screenshot_set)
data['screenshots'] = screenshots_out
else:
data['screenshots'] = None
# Contacts
if 'contacts' in data:
for contact in data['contacts']:
if 'name' not in contact:
raise serializers.ValidationError('Contact requires [name] field')
if 'email' not in contact:
raise serializers.ValidationError('Contact requires [email] field')
if 'secure_phone' not in contact:
contact['secure_phone'] = None
if 'unsecure_phone' not in contact:
contact['unsecure_phone'] = None
if 'contact_type' not in contact:
raise serializers.ValidationError('Contact requires [contact_type] field')
contact_type = contact.get('contact_type')
contact_type_name = None if contact_type is None else contact_type.get('name')
if not contact_type_name:
raise serializers.ValidationError('Contact field requires correct format')
contact_type_instance = contact_type_model_access.get_contact_type_by_name(contact_type_name)
if not contact_type_instance:
raise serializers.ValidationError('Contact Type [{}] not found'.format(contact_type_name))
# owners
owners = []
if 'owners' in data:
for owner in data['owners']:
user_dict = owner.get('user')
user_username = None if user_dict is None else user_dict.get('username')
if not user_username:
raise serializers.ValidationError('Owner field requires correct format')
owner_profile = generic_model_access.get_profile(user_username)
if not owner_profile:
raise serializers.ValidationError('Owner Profile not found')
# Don't allow user to select a security marking that is above
# their own access level
if not system_has_access_control(owner_profile.user.username, data.get('security_marking')):
raise serializers.ValidationError('Security marking too high for current owner profile')
# Don't allow 2nd-party user to be an owner of a listing
if system_anonymize_identifiable_data(owner_profile.user.username):
raise serializers.ValidationError('Permissions are invalid for current owner profile')
owners.append(owner_profile)
data['owners'] = owners
# Categories
categories = []
if 'categories' in data:
for category in data['categories']:
category_title = category.get('title')
if not category_title:
raise serializers.ValidationError('Categories field requires correct format')
category_instance = category_model_access.get_category_by_title(category_title)
if not category_instance:
raise serializers.ValidationError('Category [{}] not found'.format(category_title))
categories.append(category_instance)
data['categories'] = categories
# Intents
intents = []
if 'intents' in data:
for intent in data['intents']:
intent_action = intent.get('action')
if not intent_action:
raise serializers.ValidationError('Intent field requires correct format')
intent_instance = intent_model_access.get_intent_by_action(intent_action)
if not intent_instance:
raise serializers.ValidationError('Intent Action [{}] not found'.format(intent_action))
intents.append(intent_instance)
data['intents'] = intents
# doc urls will be created in create()
if 'doc_urls' in data:
pass
# tags will be created (if necessary) in create()
if 'tags' in data:
pass
return data