def create_listing(listing_builder_dict, object_cache):
"""
Create Listing Helper Function
10-01-2017 - Total Database Calls: 11842
10-02-2017 - Total Database Calls: 7737
"""
listing_data = listing_builder_dict['listing']
listing = models.Listing(
title=listing_data['title'],
agency=object_cache['Agency.{}'.format(listing_data['agency'])],
listing_type=object_cache['ListingType.{}'.format(
listing_data['listing_type'])],
description=listing_data['description'],
launch_url=listing_data['launch_url'].format_map(
{'DEMO_APP_ROOT': DEMO_APP_ROOT}),
version_name=listing_data['version_name'],
unique_name=listing_data['unique_name'],
small_icon=object_cache['Listing[{}].small_icon'.format(
listing_data['title'])],
large_icon=object_cache['Listing[{}].large_icon'.format(
listing_data['title'])],
banner_icon=object_cache['Listing[{}].banner_icon'.format(
listing_data['title'])],
large_banner_icon=object_cache['Listing[{}].large_banner_icon'.format(
listing_data['title'])],
what_is_new=listing_data['what_is_new'],
description_short=listing_data['description_short'],
usage_requirements=listing_data['usage_requirements'],
system_requirements=listing_data['system_requirements'],
is_enabled=listing_data['is_enabled'],
is_private=listing_data['is_private'],
is_featured=listing_data['is_featured'],
iframe_compatible=listing_data['iframe_compatible'],
security_marking=listing_data['security_marking'])
listing.save()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Contacts
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
for current_contact in listing_data['contacts']:
listing.contacts.add(
object_cache['Contact.{}'.format(current_contact)])
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Owners
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
for current_owner in listing_data['owners']:
listing.owners.add(object_cache['Profile.{}'.format(current_owner)])
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Categories
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
for current_category in listing_data['categories']:
listing.categories.add(
models.Category.objects.get(title=current_category))
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Tags
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
for current_tag in listing_data['tags']:
if object_cache.get('Tag.{}'.format(current_tag)):
current_tag_obj = object_cache['Tag.{}'.format(current_tag)]
else:
current_tag_obj, created = models.Tag.objects.get_or_create(
name=current_tag)
listing.tags.add(current_tag_obj)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Screenshots
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
for current_screenshot_entry in listing_data['screenshots']:
small_image = models.Image.create_image(
Image.open(TEST_IMG_PATH +
current_screenshot_entry['small_image']['filename']),
file_extension=current_screenshot_entry['small_image']
['filename'].split('.')[-1],
security_marking=current_screenshot_entry['small_image']
['security_marking'],
image_type=object_cache['ImageType.small_screenshot'].name)
large_image = models.Image.create_image(
Image.open(TEST_IMG_PATH +
current_screenshot_entry['large_image']['filename']),
file_extension=current_screenshot_entry['large_image']
['filename'].split('.')[-1],
security_marking=current_screenshot_entry['large_image']
['security_marking'],
image_type=object_cache['ImageType.large_screenshot'].name)
screenshot = models.Screenshot(
small_image=small_image,
large_image=large_image,
listing=listing,
description=current_screenshot_entry['description'],
order=current_screenshot_entry['order'])
screenshot.save()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Document URLs
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
for current_doc_url_entry in listing_data['doc_urls']:
current_doc_url_obj = models.DocUrl(name=current_doc_url_entry['name'],
url=current_doc_url_entry['url'],
listing=listing)
current_doc_url_obj.save()
# listing_activity
for listing_activity_entry in listing_builder_dict['listing_activity']:
listing_activity_action = listing_activity_entry['action']
listing_activity_author = object_cache['Profile.{}'.format(
listing_activity_entry['author'])]
if listing_activity_action == 'CREATED':
listing_model_access.create_listing(listing_activity_author,
listing)
elif listing_activity_action == 'SUBMITTED':
listing_model_access.submit_listing(listing_activity_author,
listing)
elif listing_activity_action == 'APPROVED_ORG':
listing_model_access.approve_listing_by_org_steward(
listing_activity_author, listing)
elif listing_activity_action == 'APPROVED':
listing_model_access.approve_listing(listing_activity_author,
listing)
return listing
def create_listing(self, listing: ListingData) -> models.Listing:
agency = self.agencies_by_title.get(listing.agency_title, None)
assert agency is not None, "Agency with title '{}' not found".format(
listing.agency_title)
listing_type = self.listing_types_by_title.get(
listing.listing_type_title, None)
assert listing_type is not None, "ListingType with title '{}' not found".format(
listing.listing_type_title)
small_icon = self.create_image(listing.small_icon)
large_icon = self.create_image(listing.large_icon)
banner_icon = self.create_image(listing.banner_icon)
large_banner_icon = self.create_image(listing.large_banner_icon)
instance = models.Listing(
title=listing.title,
description=listing.description,
description_short=listing.description_short,
launch_url=listing.launch_url,
version_name=listing.version_name,
unique_name=listing.unique_name,
what_is_new=listing.what_is_new,
usage_requirements=listing.usage_requirements,
system_requirements=listing.system_requirements,
security_marking=listing.security_marking,
is_enabled=listing.is_enabled,
is_private=listing.is_private,
is_featured=listing.is_featured,
is_exportable=listing.is_exportable,
small_icon=small_icon,
large_icon=large_icon,
banner_icon=banner_icon,
large_banner_icon=large_banner_icon,
agency=agency,
listing_type=listing_type)
instance.save()
for category_title in listing.category_titles:
category = self.categories_by_title.get(category_title, None)
assert category is not None, "Category with title '{}' not found".format(
category_title)
instance.categories.add(category)
for contact_email in listing.contact_emails:
contact = self.contacts_by_email.get(contact_email, None)
assert contact is not None, "Contact with email '{}' not found".format(
contact_email)
instance.contacts.add(contact)
for custom_field_value in listing.custom_field_values:
self.create_custom_field_value(instance, custom_field_value)
for intent_action in listing.intent_actions:
intent = self.intents_by_action.get(intent_action, None)
assert intent is not None, "Intent with action '{}' not found".format(
intent_action)
instance.intents.add(intent)
for owner_username in listing.owner_usernames:
owner = self.profiles_by_username.get(owner_username, None)
assert owner is not None, "Profile with username '{}' not found".format(
owner_username)
instance.owners.add(owner)
for tag_name in listing.tag_names:
tag = self.tags_by_name.get(tag_name, None)
assert tag is not None, "Tag with name '{}' not found".format(
tag_name)
instance.tags.add(tag)
for screenshot in listing.screenshots:
self.create_screenshot(instance, screenshot)
for doc_url in listing.doc_urls:
self.create_doc_url(instance, doc_url)
for activity in listing.activities:
author = self.profiles_by_username.get(activity.author_username,
None)
assert author is not None, "Profile with username '{}' not found".format(
activity.author_username)
if activity.action == 'CREATED':
listing_model_access.create_listing(author, instance)
elif activity.action == 'SUBMITTED':
listing_model_access.submit_listing(author, instance)
elif activity.action == 'APPROVED_ORG':
listing_model_access.approve_listing_by_org_steward(
author, instance)
elif activity.action == 'APPROVED':
listing_model_access.approve_listing(author, instance)
return instance
def create(self, validated_data):
# logger.debug('inside ListingSerializer.create', extra={'request':self.context.get('request')})
title = validated_data['title']
user = generic_model_access.get_profile(
self.context['request'].user.username)
logger.info('creating listing {0!s} for user {1!s}'.format(
title, user.user.username),
extra={'request': self.context.get('request')})
# TODO required_listings
listing = models.Listing(
title=title,
agency=validated_data['agency'],
description=validated_data['description'],
launch_url=validated_data['launch_url'],
version_name=validated_data['version_name'],
unique_name=validated_data['unique_name'],
what_is_new=validated_data['what_is_new'],
description_short=validated_data['description_short'],
requirements=validated_data['requirements'],
security_marking=validated_data['security_marking'],
listing_type=validated_data['listing_type'],
is_private=validated_data['is_private'])
image_keys = [
'small_icon', 'large_icon', 'banner_icon', 'large_banner_icon'
]
for image_key in image_keys:
if validated_data[image_key]:
new_value_image = image_model_access.get_image_by_id(
validated_data[image_key].get('id'))
if new_value_image is None:
raise errors.InvalidInput(
'Error while saving, can not find image by id')
if image_key == 'small_icon':
listing.small_icon = new_value_image
elif image_key == 'large_icon':
listing.large_icon = new_value_image
elif image_key == 'banner_icon':
listing.banner_icon = new_value_image
elif image_key == 'large_banner_icon':
listing.large_banner_icon = new_value_image
listing.save()
if validated_data.get('contacts') is not None:
for contact in validated_data['contacts']:
contact_type_instance = contact_type_model_access.get_contact_type_by_name(
contact['contact_type']['name'])
new_contact, created = models.Contact.objects.get_or_create(
name=contact['name'],
email=contact['email'],
secure_phone=contact['secure_phone'],
unsecure_phone=contact['unsecure_phone'],
organization=contact.get('organization', None),
contact_type=contact_type_instance)
new_contact.save()
listing.contacts.add(new_contact)
if validated_data.get('owners') is not None:
if validated_data['owners']:
for owner in validated_data['owners']:
listing.owners.add(owner)
else:
# if no owners are specified, just add the current user
listing.owners.add(user)
if validated_data.get('categories') is not None:
for category in validated_data['categories']:
listing.categories.add(category)
# tags will be automatically created if necessary
if validated_data.get('tags') is not None:
for tag in validated_data['tags']:
obj, created = models.Tag.objects.get_or_create(
name=tag['name'])
listing.tags.add(obj)
if validated_data.get('intents') is not None:
for intent in validated_data['intents']:
listing.intents.add(intent)
# doc_urls will be automatically created
if validated_data.get('doc_urls') is not None:
for d in validated_data['doc_urls']:
doc_url = models.DocUrl(name=d['name'],
url=d['url'],
listing=listing)
doc_url.save()
# screenshots will be automatically created
if validated_data.get('screenshots') is not None:
for screenshot_dict in validated_data['screenshots']:
screenshot = models.Screenshot(
small_image=image_model_access.get_image_by_id(
screenshot_dict['small_image']['id']),
large_image=image_model_access.get_image_by_id(
screenshot_dict['large_image']['id']),
listing=listing)
screenshot.save()
# create a new activity
model_access.create_listing(user, listing)
return listing
def create_listing(listing_builder_dict):
"""
Create Listing Helper Function
"""
listing_data = listing_builder_dict['listing']
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Icons
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
small_icon = models.Image.create_image(
Image.open(TEST_IMG_PATH + listing_data['small_icon']['filename']),
file_extension=listing_data['small_icon']['filename'].split('.')[-1],
security_marking=listing_data['small_icon']['security_marking'],
image_type=models.ImageType.objects.get(name='small_icon').name)
large_icon = models.Image.create_image(
Image.open(TEST_IMG_PATH + listing_data['large_icon']['filename']),
file_extension=listing_data['large_icon']['filename'].split('.')[-1],
security_marking=listing_data['large_icon']['security_marking'],
image_type=models.ImageType.objects.get(name='large_icon').name)
banner_icon = models.Image.create_image(
Image.open(TEST_IMG_PATH + listing_data['banner_icon']['filename']),
file_extension=listing_data['banner_icon']['filename'].split('.')[-1],
security_marking=listing_data['banner_icon']['security_marking'],
image_type=models.ImageType.objects.get(name='banner_icon').name)
large_banner_icon = models.Image.create_image(
Image.open(TEST_IMG_PATH +
listing_data['large_banner_icon']['filename']),
file_extension=listing_data['large_banner_icon']['filename'].split(
'.')[-1],
security_marking=listing_data['large_banner_icon']['security_marking'],
image_type=models.ImageType.objects.get(name='large_banner_icon').name)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Listing
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
listing = models.Listing(
title=listing_data['title'],
agency=models.Agency.objects.get(short_name=listing_data['agency']),
listing_type=models.ListingType.objects.get(
title=listing_data['listing_type']),
description=listing_data['description'],
launch_url=listing_data['launch_url'].format_map(
{'DEMO_APP_ROOT': DEMO_APP_ROOT}),
version_name=listing_data['version_name'],
unique_name=listing_data['unique_name'],
small_icon=small_icon,
large_icon=large_icon,
banner_icon=banner_icon,
large_banner_icon=large_banner_icon,
what_is_new=listing_data['what_is_new'],
description_short=listing_data['description_short'],
requirements=listing_data['requirements'],
is_enabled=listing_data['is_enabled'],
is_private=listing_data['is_private'],
is_featured=listing_data['is_featured'],
iframe_compatible=listing_data['iframe_compatible'],
security_marking=listing_data['security_marking'])
listing.save()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Contacts
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
for current_contact in listing_data['contacts']:
listing.contacts.add(models.Contact.objects.get(email=current_contact))
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Owners
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
for current_owner in listing_data['owners']:
listing.owners.add(
models.Profile.objects.get(user__username=current_owner))
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Categories
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
for current_category in listing_data['categories']:
listing.categories.add(
models.Category.objects.get(title=current_category))
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Tags
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
for current_tag in listing_data['tags']:
current_tag_obj, created = models.Tag.objects.get_or_create(
name=current_tag)
listing.tags.add(current_tag_obj)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Screenshots
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
for current_screenshot_entry in listing_data['screenshots']:
small_image = models.Image.create_image(
Image.open(TEST_IMG_PATH +
current_screenshot_entry['small_image']['filename']),
file_extension=current_screenshot_entry['small_image']
['filename'].split('.')[-1],
security_marking=current_screenshot_entry['small_image']
['security_marking'],
image_type=models.ImageType.objects.get(
name='small_screenshot').name)
large_image = models.Image.create_image(
Image.open(TEST_IMG_PATH +
current_screenshot_entry['large_image']['filename']),
file_extension=current_screenshot_entry['large_image']
['filename'].split('.')[-1],
security_marking=current_screenshot_entry['large_image']
['security_marking'],
image_type=models.ImageType.objects.get(
name='large_screenshot').name)
screenshot = models.Screenshot(
small_image=small_image,
large_image=large_image,
listing=listing,
description=current_screenshot_entry['description'],
order=current_screenshot_entry['order'])
screenshot.save()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Document URLs
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
for current_doc_url_entry in listing_data['doc_urls']:
current_doc_url_obj = models.DocUrl(name=current_doc_url_entry['name'],
url=current_doc_url_entry['url'],
listing=listing)
current_doc_url_obj.save()
# listing_activity
for listing_activity_entry in listing_builder_dict['listing_activity']:
listing_activity_action = listing_activity_entry['action']
listing_activity_author = models.Profile.objects.get(
user__username=listing_activity_entry['author'])
if listing_activity_action == 'CREATED':
listing_model_access.create_listing(listing_activity_author,
listing)
elif listing_activity_action == 'SUBMITTED':
listing_model_access.submit_listing(listing_activity_author,
listing)
elif listing_activity_action == 'APPROVED_ORG':
listing_model_access.approve_listing_by_org_steward(
listing_activity_author, listing)
elif listing_activity_action == 'APPROVED':
listing_model_access.approve_listing(listing_activity_author,
listing)
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Reviews
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# listing_review_batch
create_listing_review_batch(listing,
listing_builder_dict['listing_review_batch'])
return listing
def migrate_listing(category_mapper, agency_mapper, type_mapper,
contact_type_mapper, profile_mapper):
logging.debug('migrating listings')
columns = get_columns('listing')
logging.debug('listing columns: {0!s}'.format(columns))
# ['id', 'version', 'agency_id', 'approval_status', 'approved_date', 'avg_rate',
# 'created_by_id', 'created_date', 'description', 'description_short',
# 'edited_by_id', 'edited_date', 'small_icon_id', 'large_icon_id',
# 'banner_icon_id', 'featured_banner_icon_id', 'is_enabled', 'is_featured',
# 'last_activity_id', 'launch_url', 'requirements', 'title', 'total_comments',
# 'total_rate1', 'total_rate2', 'total_rate3', 'total_rate4', 'total_rate5',
# 'total_votes', 'type_id', 'uuid', 'version_name', 'what_is_new',
# 'width', 'singleton', 'height']
assert columns[0] == 'id'
assert columns[1] == 'version'
assert columns[2] == 'agency_id'
assert columns[3] == 'approval_status'
assert columns[4] == 'approved_date'
assert columns[5] == 'avg_rate'
assert columns[6] == 'created_by_id'
assert columns[7] == 'created_date'
assert columns[8] == 'description'
assert columns[9] == 'description_short'
assert columns[10] == 'edited_by_id'
assert columns[11] == 'edited_date'
assert columns[12] == 'small_icon_id'
assert columns[13] == 'large_icon_id'
assert columns[14] == 'banner_icon_id'
assert columns[15] == 'featured_banner_icon_id'
assert columns[16] == 'is_enabled'
assert columns[17] == 'is_featured'
assert columns[18] == 'last_activity_id'
assert columns[19] == 'launch_url'
assert columns[20] == 'requirements'
assert columns[21] == 'title'
assert columns[22] == 'total_comments'
assert columns[23] == 'total_rate1'
assert columns[24] == 'total_rate2'
assert columns[25] == 'total_rate3'
assert columns[26] == 'total_rate4'
assert columns[27] == 'total_rate5'
assert columns[28] == 'total_votes'
assert columns[29] == 'type_id'
assert columns[30] == 'uuid'
assert columns[31] == 'version_name'
assert columns[32] == 'what_is_new'
assert columns[33] == 'width'
assert columns[34] == 'singleton'
assert columns[35] == 'height'
values = get_values('listing', len(columns))
# logging.debug('listing values: %s' % values)
logging.info('Listings to migrate: {0!s}'.format(len(values)))
# map old ids to new ones for future migrations: {'<old_id>': '<new_id>'}
listing_mapper = {}
logging.info('==========================')
for i in values:
try:
old_id = i[0]
agency = models.Agency.objects.get(id=agency_mapper[i[2]])
title = i[21]
# approval_status should be a 1-1 mapping
approval_status = i[3]
approved_date = get_date_from_str(i[4])
avg_rate = i[5]
created_by_id = i[6]
created_date = get_date_from_str(i[7])
description = i[8]
description_short = i[9]
edited_by_id = i[10]
edited_date = get_date_from_str(i[11])
small_icon_id = i[12]
small_icon = migrate_image(small_icon_id, 'small_icon')
large_icon_id = i[13]
large_icon = migrate_image(large_icon_id, 'large_icon')
banner_icon_id = i[14]
banner_icon = migrate_image(banner_icon_id, 'banner_icon')
featured_banner_icon_id = i[15]
large_banner_icon = migrate_image(featured_banner_icon_id,
'large_banner_icon')
is_enabled = i[16]
is_featured = i[17]
last_activity_id = i[18]
launch_url = i[19]
requirements = i[20]
total_comments = i[22]
total_rate1 = i[23]
total_rate2 = i[24]
total_rate3 = i[25]
total_rate4 = i[26]
total_rate5 = i[27]
total_votes = i[28]
listing_type = models.ListingType(id=type_mapper[i[29]])
uuid = i[30]
version_name = i[31]
what_is_new = i[32]
iframe_compatible = not i[34]
logging.info('Adding listing title: {0!s}'.format((title)))
# TODO: unique_name?
listing = models.Listing(title=title,
agency=agency,
approval_status=approval_status,
approved_date=approved_date,
edited_date=edited_date,
description=description,
description_short=description_short,
is_enabled=is_enabled,
is_featured=is_featured,
launch_url=launch_url,
listing_type=listing_type,
version_name=version_name,
what_is_new=what_is_new,
iframe_compatible=iframe_compatible,
requirements=requirements,
small_icon=small_icon,
large_icon=large_icon,
banner_icon=banner_icon,
large_banner_icon=large_banner_icon,
avg_rate=avg_rate,
total_votes=total_votes,
total_rate5=total_rate5,
total_rate4=total_rate4,
total_rate3=total_rate3,
total_rate2=total_rate2,
total_rate1=total_rate1,
total_reviews=total_comments,
security_marking=DEFAULT_SECURITY_MARKING)
listing.save()
listing_mapper[old_id] = str(listing.id)
except Exception as e:
logging.error('Error processing listing {0!s}: {1!s}'.format(
title, str(e)))
return listing_mapper